Loading...
--- /dev/null
+++ Libc/Libc-262/ppc/gen/bcopy.s
@@ -0,0 +1,1159 @@
+/*
+ * Copyright (c) 2002 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_LICENSE_HEADER_START@
+ *
+ * The contents of this file constitute Original Code as defined in and
+ * are subject to the Apple Public Source License Version 1.1 (the
+ * "License"). You may not use this file except in compliance with the
+ * License. Please obtain a copy of the License at
+ * http://www.apple.com/publicsource and read it before using this file.
+ *
+ * This Original Code and all software distributed under the License are
+ * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT. Please see the
+ * License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * @APPLE_LICENSE_HEADER_END@
+ */
+/* =======================================
+ * BCOPY, MEMCPY, and MEMMOVE for Mac OS X
+ * =======================================
+ *
+ * Version of 6/17/2002, for G3, G4, and G4+.
+ *
+ * There are many paths through this code, depending on length, reverse/forward,
+ * processor type, and alignment. We use reverse paths only when the operands
+ * overlap and the destination is higher than the source. They are not quite as
+ * fast as the forward paths.
+ *
+ * Judicious use of DCBTs, just far enough ahead to minimize waiting, is critical in
+ * the inner loops for long operands. DST is less effective than DCBT, because it
+ * can get out of sync with the inner loop. DCBTST is usually not a win, so we
+ * don't use it except during initialization when we're not using the LSU.
+ * We don't DCBT on G3, which only handles one load miss at a time.
+ *
+ * We don't use DCBZ, because it takes an alignment exception on uncached memory
+ * like frame buffers. Bcopy to frame buffers must work. This hurts G3 in the
+ * cold-cache case, but G4 can use DCBA (which does not take alignment exceptions.)
+ *
+ * Using DCBA on G4 is a tradeoff. For the cold-cache case it can be a big win,
+ * since it avoids the read of destination cache lines. But for the hot-cache case
+ * it is always slower, because of the cycles spent needlessly zeroing data. Some
+ * machines store-gather and can cancel the read if all bytes of a line are stored,
+ * others cannot. Unless explicitly told which is better, we time loops with and
+ * without DCBA and use the fastest. Note that we never DCBA in reverse loops,
+ * since by definition they are overlapped so dest lines will be in the cache.
+ *
+ * For longer operands we use an 8-element branch table, based on the CPU type,
+ * to select the appropriate inner loop. The branch table is indexed as follows:
+ *
+ * bit 10000 set if a Reverse move is required
+ * bits 01100 set on the relative operand alignment: 0=unaligned, 1=word,
+ * 2=doubleword, and 3=quadword.
+ *
+ * By "relatively" n-byte aligned, we mean the source and destination are a multiple
+ * of n bytes apart (they need not be absolutely aligned.)
+ *
+ * The branch table for the running CPU type is pointed to by LBranchTablePtr.
+ * Initially, LBranchtablePtr points to G3's table, since that is the lowest
+ * common denominator that will run on any CPU. Later, pthread initialization
+ * sets up the _cpu_capabilities vector and calls _bcopy_initialize, which sets
+ * up the correct pointer for the running CPU.
+ *
+ * We distinguish between "short", "medium", and "long" operands:
+ * short (<= 32 bytes) most common case, minimum path length is important
+ * medium (> 32, < kLong) too short for Altivec or use of cache ops like DCBA
+ * long (>= kLong) long enough for cache ops and to amortize use of Altivec
+ *
+ * WARNING: kLong must be >=96, due to implicit assumptions about operand length.
+ */
+#define kLong 96
+
+/* Register usage. Note we use R2, so this code will not run in a PEF/CFM
+ * environment. Note also the rather delicate way we assign multiple uses
+ * to the same register. Beware.
+ *
+ * r0 = "w7" or "r0" (NB: cannot use r0 for any constant such as "c16")
+ * r2 = "w8" or VRSave ("rv")
+ * r3 = not used, as memcpy and memmove return 1st parameter as a value
+ * r4 = source ptr ("rs")
+ * r5 = count of bytes to move ("rc")
+ * r6 = "w1", "c16", or "cm17"
+ * r7 = "w2", "c32", or "cm33"
+ * r8 = "w3", "c48", or "cm49"
+ * r9 = "w4", "c64", or "cm1"
+ * r10 = "w5", "c96", or "cm97"
+ * r11 = "w6", "c128", "cm129", or return address ("ra")
+ * r12 = destination ptr ("rd")
+ * f0-f8 = used for moving 8-byte aligned data
+ * v0 = permute vector ("vp")
+ * v1-v4 = qw's loaded from source ("v1", "v2", "v3", and "v4")
+ * v5-v7 = permuted qw's ("vx", "vy", and "vz")
+ */
+#define rs r4
+#define rd r12
+#define rc r5
+#define ra r11
+#define rv r2
+
+#define w1 r6
+#define w2 r7
+#define w3 r8
+#define w4 r9
+#define w5 r10
+#define w6 r11
+#define w7 r0
+#define w8 r2
+
+#define c16 r6
+#define cm17 r6
+#define c32 r7
+#define cm33 r7
+#define c48 r8
+#define cm49 r8
+#define c64 r9
+#define cm1 r9
+#define c96 r10
+#define cm97 r10
+#define c128 r11
+#define cm129 r11
+
+#define vp v0
+#define vx v5
+#define vy v6
+#define vz v7
+
+#define VRSave 256
+
+#include <architecture/ppc/asm_help.h>
+
+// The branch tables, 8 entries per CPU type.
+// NB: we depend on 5 low-order 0s in the address of branch tables.
+
+ .data
+ .align 5 // must be 32-byte aligned
+
+ // G3 (the default CPU type)
+
+LG3:
+ .long LForwardWord // 000: forward, unaligned
+ .long LForwardFloat // 001: forward, 4-byte aligned
+ .long LForwardFloat // 010: forward, 8-byte aligned
+ .long LForwardFloat // 011: forward, 16-byte aligned
+ .long LReverseWord // 100: reverse, unaligned
+ .long LReverseFloat // 101: reverse, 4-byte aligned
+ .long LReverseFloat // 110: reverse, 8-byte aligned
+ .long LReverseFloat // 111: reverse, 16-byte aligned
+
+ // G4s that benefit from DCBA.
+
+LG4UseDcba:
+ .long LForwardVecUnal32Dcba // 000: forward, unaligned
+ .long LForwardVecUnal32Dcba // 001: forward, 4-byte aligned
+ .long LForwardVecUnal32Dcba // 010: forward, 8-byte aligned
+ .long LForwardVecAlig32Dcba // 011: forward, 16-byte aligned
+ .long LReverseVectorUnal32 // 100: reverse, unaligned
+ .long LReverseVectorUnal32 // 101: reverse, 4-byte aligned
+ .long LReverseVectorUnal32 // 110: reverse, 8-byte aligned
+ .long LReverseVectorAligned32 // 111: reverse, 16-byte aligned
+
+ // G4s that should not use DCBA.
+
+LG4NoDcba:
+ .long LForwardVecUnal32NoDcba // 000: forward, unaligned
+ .long LForwardVecUnal32NoDcba // 001: forward, 4-byte aligned
+ .long LForwardVecUnal32NoDcba // 010: forward, 8-byte aligned
+ .long LForwardVecAlig32NoDcba // 011: forward, 16-byte aligned
+ .long LReverseVectorUnal32 // 100: reverse, unaligned
+ .long LReverseVectorUnal32 // 101: reverse, 4-byte aligned
+ .long LReverseVectorUnal32 // 110: reverse, 8-byte aligned
+ .long LReverseVectorAligned32 // 111: reverse, 16-byte aligned
+
+
+// Pointer to the 8-element branch table for running CPU type:
+
+LBranchTablePtr:
+ .long LG3 // default to G3 until "bcopy_initialize" called
+
+
+// The CPU capability vector, initialized in pthread_init().
+// "_bcopy_initialize" uses this to set up LBranchTablePtr:
+
+ .globl __cpu_capabilities
+__cpu_capabilities:
+ .long 0
+
+// Bit definitions for _cpu_capabilities:
+
+#define kHasAltivec 0x01
+#define k64Bit 0x02
+#define kCache32 0x04
+#define kCache64 0x08
+#define kCache128 0x10
+#define kUseDcba 0x20
+#define kNoDcba 0x40
+
+
+.text
+.globl _bcopy
+.globl _memcpy
+.globl _memmove
+.globl __bcopy_initialize
+
+
+// Main entry points.
+
+ .align 5
+_bcopy: // void bcopy(const void *src, void *dst, size_t len)
+ mr r10,r3 // reverse source and dest ptrs, to be like memcpy
+ mr r3,r4
+ mr r4,r10
+_memcpy: // void* memcpy(void *dst, void *src, size_t len)
+_memmove: // void* memmove(void *dst, const void *src, size_t len)
+ cmplwi cr7,rc,32 // length <= 32 bytes?
+ sub. w1,r3,rs // must move in reverse if (rd-rs)<rc, set cr0 on sou==dst
+ dcbt 0,rs // touch in the first line of source
+ cmplw cr6,w1,rc // set cr6 blt iff we must move reverse
+ cmplwi cr1,rc,kLong-1 // set cr1 bgt if long
+ mr rd,r3 // must leave r3 alone, it is return value for memcpy etc
+ bgt- cr7,LMedium // longer than 32 bytes
+ dcbtst 0,rd // touch in destination
+ beq- cr7,LMove32 // special case moves of 32 bytes
+ blt- cr6,LShortReverse0
+
+// Forward short operands. This is the most frequent case, so it is inline.
+// We also end up here to xfer the last 0-31 bytes of longer operands.
+
+LShort: // WARNING: can fall into this routine
+ andi. r0,rc,0x10 // test bit 27 separately (sometimes faster than a mtcrf)
+ mtcrf 0x01,rc // move rest of length to cr7
+ beq 1f // quadword to move?
+ lwz w1,0(rs)
+ lwz w2,4(rs)
+ lwz w3,8(rs)
+ lwz w4,12(rs)
+ addi rs,rs,16
+ stw w1,0(rd)
+ stw w2,4(rd)
+ stw w3,8(rd)
+ stw w4,12(rd)
+ addi rd,rd,16
+1:
+LShort16: // join here to xfer 0-15 bytes
+ bf 28,2f // doubleword?
+ lwz w1,0(rs)
+ lwz w2,4(rs)
+ addi rs,rs,8
+ stw w1,0(rd)
+ stw w2,4(rd)
+ addi rd,rd,8
+2:
+ bf 29,3f // word?
+ lwz w1,0(rs)
+ addi rs,rs,4
+ stw w1,0(rd)
+ addi rd,rd,4
+3:
+ bf 30,4f // halfword to move?
+ lhz w1,0(rs)
+ addi rs,rs,2
+ sth w1,0(rd)
+ addi rd,rd,2
+4:
+ bflr 31 // skip if no odd byte
+ lbz w1,0(rs)
+ stb w1,0(rd)
+ blr
+
+
+// Handle short reverse operands, up to kShort in length.
+// This is also used to transfer the last 0-31 bytes of longer operands.
+
+LShortReverse0:
+ add rs,rs,rc // adjust ptrs for reverse move
+ add rd,rd,rc
+LShortReverse:
+ andi. r0,rc,0x10 // test bit 27 separately (sometimes faster than a mtcrf)
+ mtcrf 0x01,rc // move rest of length to cr7
+ beq 1f // quadword to move?
+ lwz w1,-4(rs)
+ lwz w2,-8(rs)
+ lwz w3,-12(rs)
+ lwzu w4,-16(rs)
+ stw w1,-4(rd)
+ stw w2,-8(rd)
+ stw w3,-12(rd)
+ stwu w4,-16(rd)
+1:
+LShortReverse16: // join here to xfer 0-15 bytes and return
+ bf 28,2f // doubleword?
+ lwz w1,-4(rs)
+ lwzu w2,-8(rs)
+ stw w1,-4(rd)
+ stwu w2,-8(rd
+2:
+ bf 29,3f // word?
+ lwzu w1,-4(rs)
+ stwu w1,-4(rd)
+3:
+ bf 30,4f // halfword to move?
+ lhzu w1,-2(rs)
+ sthu w1,-2(rd)
+4:
+ bflr 31 // done if no odd byte
+ lbz w1,-1(rs) // no update
+ stb w1,-1(rd)
+ blr
+
+
+// Special case for 32-byte moves. Too long for LShort, too common for LMedium.
+
+LMove32:
+ lwz w1,0(rs)
+ lwz w2,4(rs)
+ lwz w3,8(rs)
+ lwz w4,12(rs)
+ lwz w5,16(rs)
+ lwz w6,20(rs)
+ lwz w7,24(rs)
+ lwz w8,28(rs)
+ stw w1,0(rd)
+ stw w2,4(rd)
+ stw w3,8(rd)
+ stw w4,12(rd)
+ stw w5,16(rd)
+ stw w6,20(rd)
+ stw w7,24(rd)
+ stw w8,28(rd)
+LExit:
+ blr
+
+
+// Medium length operands (32 < rc < kLong.) These loops run on all CPUs, as the
+// operands are not long enough to bother with the branch table, using cache ops, or
+// Altivec. We word align the source, not the dest as we do for long operands,
+// since doing so is faster on G4+ and probably beyond, we never DCBA on medium-length
+// operands, and the opportunity to cancel reads of dest cache lines is limited.
+// w1 = (rd-rs), used to check for alignment
+// cr0 = set on (rd-rs)
+// cr1 = bgt if long operand
+// cr6 = blt if reverse move
+
+LMedium:
+ dcbtst 0,rd // touch in 1st line of destination
+ rlwinm r0,w1,0,29,31 // r0 <- ((rd-rs) & 7), ie 0 if doubleword aligned
+ beq- LExit // early exit if (rs==rd), avoiding use of "beqlr"
+ neg w2,rs // we align source, not dest, and assume forward
+ cmpwi cr5,r0,0 // set cr5 beq if doubleword aligned
+ bgt- cr1,LLong // handle long operands
+ andi. w3,w2,3 // W3 <- #bytes to word-align source
+ blt- cr6,LMediumReverse // handle reverse move
+ lwz w1,0(rs) // pre-fetch first 4 bytes of source
+ beq- cr5,LMediumAligned // operands are doubleword aligned
+ sub rc,rc,w3 // adjust count for alignment
+ mtcrf 0x01,rc // remaining byte count (0-15) to cr7 for LShort16
+ srwi w4,rc,4 // w4 <- number of 16-byte chunks to xfer (>=1)
+ mtctr w4 // prepare loop count
+ beq+ 2f // source already aligned
+
+ lwzx w2,w3,rs // get 1st aligned word (which we might partially overwrite)
+ add rs,rs,w3 // word-align source ptr
+ stw w1,0(rd) // store all (w3) bytes at once to avoid a loop
+ add rd,rd,w3
+ mr w1,w2 // first aligned word to w1
+ b 2f
+
+ .align 4 // align inner loops
+1: // loop over 16-byte chunks
+ lwz w1,0(rs)
+2:
+ lwz w2,4(rs)
+ lwz w3,8(rs)
+ lwz w4,12(rs)
+ addi rs,rs,16
+ stw w1,0(rd)
+ stw w2,4(rd)
+ stw w3,8(rd)
+ stw w4,12(rd)
+ addi rd,rd,16
+ bdnz 1b
+
+ b LShort16
+
+
+// Medium, doubleword aligned. We use floating point. Note that G4+ has bigger latencies
+// and reduced throughput for floating pt loads and stores; future processors will probably
+// have even worse lfd/stfd performance. We use it here because it is so important for G3,
+// and not slower for G4+. But we only do so for doubleword aligned operands, whereas the
+// G3-only long operand loops use floating pt even for word-aligned operands.
+// w2 = neg(rs)
+// w1 = first 4 bytes of source
+
+LMediumAligned:
+ andi. w3,w2,7 // already aligned?
+ sub rc,rc,w3 // adjust count by 0-7 bytes
+ lfdx f0,rs,w3 // pre-fetch first aligned source doubleword
+ srwi w4,rc,5 // get count of 32-byte chunks (might be 0 if unaligned)
+ mtctr w4
+ beq- LForwardFloatLoop1 // already aligned
+
+ cmpwi w4,0 // are there any 32-byte chunks to xfer?
+ lwz w2,4(rs) // get 2nd (unaligned) source word
+ add rs,rs,w3 // doubleword align source pointer
+ stw w1,0(rd) // store first 8 bytes of source to align...
+ stw w2,4(rd) // ...which could overwrite source
+ add rd,rd,w3 // doubleword align destination
+ bne+ LForwardFloatLoop1 // at least 1 chunk, so enter loop
+
+ subi rc,rc,8 // unfortunate degenerate case: no chunks to xfer
+ stfd f0,0(rd) // must store f1 since source might have been overwriten
+ addi rs,rs,8
+ addi rd,rd,8
+ b LShort
+
+
+// Medium reverse moves. This loop runs on all processors.
+
+LMediumReverse:
+ add rs,rs,rc // point to other end of operands when in reverse
+ add rd,rd,rc
+ andi. w3,rs,3 // w3 <- #bytes to word align source
+ lwz w1,-4(rs) // pre-fetch 1st 4 bytes of source
+ sub rc,rc,w3 // adjust count
+ srwi w4,rc,4 // get count of 16-byte chunks (>=1)
+ mtcrf 0x01,rc // remaining byte count (0-15) to cr7 for LShortReverse16
+ mtctr w4 // prepare loop count
+ beq+ 2f // source already aligned
+
+ sub rs,rs,w3 // word-align source ptr
+ lwz w2,-4(rs) // get 1st aligned word which we may overwrite
+ stw w1,-4(rd) // store all 4 bytes to align without a loop
+ sub rd,rd,w3
+ mr w1,w2 // shift 1st aligned source word to w1
+ b 2f
+
+1:
+ lwz w1,-4(rs)
+2:
+ lwz w2,-8(rs)
+ lwz w3,-12(rs)
+ lwzu w4,-16(rs)
+ stw w1,-4(rd)
+ stw w2,-8(rd)
+ stw w3,-12(rd)
+ stwu w4,-16(rd)
+ bdnz 1b
+
+ b LShortReverse16
+
+
+// Long operands. Use branch table to decide which loop to use.
+// w1 = (rd-rs), used to determine alignment
+
+LLong:
+ xor w4,w1,rc // we must move reverse if (rd-rs)<rc
+ mflr ra // save return address
+ rlwinm w5,w1,1,27,30 // w5 <- ((w1 & 0xF) << 1)
+ bcl 20,31,1f // use reserved form to get our location
+1:
+ mflr w3 // w3 == addr(1b)
+ lis w8,0x0408 // load a 16 element, 2-bit array into w8...
+ cntlzw w4,w4 // find first difference between (rd-rs) and rc
+ addis w2,w3,ha16(LBranchTablePtr-1b)
+ ori w8,w8,0x040C // ...used to map w5 to alignment encoding (ie, to 0-3)
+ lwz w2,lo16(LBranchTablePtr-1b)(w2) // w2 <- branch table address
+ slw w4,rc,w4 // bit 0 of w4 set iff (rd-rs)<rc
+ rlwnm w5,w8,w5,28,29 // put alignment encoding in bits 01100 of w5
+ rlwimi w2,w4,5,27,27 // put reverse bit in bit 10000 of branch table address
+ lwzx w3,w2,w5 // w3 <- load loop address from branch table
+ neg w1,rd // start to compute destination alignment
+ mtctr w3
+ andi. r0,w1,0x1F // r0 <- bytes req'd to 32-byte align dest (if forward move)
+ bctr // NB: r0/cr0 and w1 are passed as parameters
+
+
+// G3, forward, long, unaligned.
+// w1 = neg(rd)
+
+LForwardWord:
+ andi. w3,w1,3 // W3 <- #bytes to word-align destination
+ mtlr ra // restore return address
+ sub rc,rc,w3 // adjust count for alignment
+ srwi r0,rc,5 // number of 32-byte chunks to xfer (>=1)
+ mtctr r0 // prepare loop count
+ beq+ 1f // dest already aligned
+
+ lwz w2,0(rs) // get first 4 bytes of source
+ lwzx w1,w3,rs // get source bytes we might overwrite
+ add rs,rs,w3 // adjust source ptr
+ stw w2,0(rd) // store all 4 bytes to avoid a loop
+ add rd,rd,w3 // word-align destination
+ b 2f
+1:
+ lwz w1,0(rs)
+2:
+ lwz w2,4(rs)
+ lwz w3,8(rs)
+ lwz w4,12(rs)
+ lwz w5,16(rs)
+ lwz w6,20(rs)
+ lwz w7,24(rs)
+ lwz w8,28(rs)
+ addi rs,rs,32
+ stw w1,0(rd)
+ stw w2,4(rd)
+ stw w3,8(rd)
+ stw w4,12(rd)
+ stw w5,16(rd)
+ stw w6,20(rd)
+ stw w7,24(rd)
+ stw w8,28(rd)
+ addi rd,rd,32
+ bdnz 1b
+
+ b LShort
+
+
+// G3, forward, long, word aligned. We use floating pt even when only word aligned.
+// w1 = neg(rd)
+
+LForwardFloat:
+ andi. w3,w1,7 // W3 <- #bytes to doubleword-align destination
+ mtlr ra // restore return address
+ sub rc,rc,w3 // adjust count for alignment
+ srwi r0,rc,5 // number of 32-byte chunks to xfer (>=1)
+ mtctr r0 // prepare loop count
+ beq LForwardFloatLoop // dest already aligned
+
+ lwz w1,0(rs) // get first 8 bytes of source
+ lwz w2,4(rs)
+ lfdx f0,w3,rs // get source bytes we might overwrite
+ add rs,rs,w3 // word-align source ptr
+ stw w1,0(rd) // store all 8 bytes to avoid a loop
+ stw w2,4(rd)
+ add rd,rd,w3
+ b LForwardFloatLoop1
+
+ .align 4 // align since this loop is executed by G4s too
+LForwardFloatLoop:
+ lfd f0,0(rs)
+LForwardFloatLoop1: // enter here from LMediumAligned and above
+ lfd f1,8(rs)
+ lfd f2,16(rs)
+ lfd f3,24(rs)
+ addi rs,rs,32
+ stfd f0,0(rd)
+ stfd f1,8(rd)
+ stfd f2,16(rd)
+ stfd f3,24(rd)
+ addi rd,rd,32
+ bdnz LForwardFloatLoop
+
+ b LShort
+
+
+// G4 Forward, long, 16-byte aligned, 32-byte cache ops, use DCBA and DCBT.
+// r0/cr0 = #bytes to 32-byte align
+
+LForwardVecAlig32Dcba:
+ bnel+ LAlign32 // align destination iff necessary
+ bl LPrepareForwardVectors
+ mtlr ra // restore return address before loading c128
+ li c128,128
+ b 1f // enter aligned loop
+
+ .align 5 // long loop heads should be at least 16-byte aligned
+1: // loop over aligned 64-byte chunks
+ dcbt c96,rs // pre-fetch three cache lines ahead
+ dcbt c128,rs // and four
+ lvx v1,0,rs
+ lvx v2,c16,rs
+ lvx v3,c32,rs
+ lvx v4,c48,rs
+ addi rs,rs,64
+ dcba 0,rd // avoid read of destination cache lines
+ stvx v1,0,rd
+ stvx v2,c16,rd
+ dcba c32,rd
+ stvx v3,c32,rd
+ stvx v4,c48,rd
+ addi rd,rd,64
+ bdnz 1b
+
+LForwardVectorAlignedEnd: // r0/cr0=#quadwords, rv=VRSave, cr7=low 4 bits of rc, cr6 set on cr7
+ beq- 3f // no leftover quadwords
+ mtctr r0
+2: // loop over remaining quadwords (1-7)
+ lvx v1,0,rs
+ addi rs,rs,16
+ stvx v1,0,rd
+ addi rd,rd,16
+ bdnz 2b
+3:
+ mtspr VRSave,rv // restore bitmap of live vr's
+ bne cr6,LShort16 // handle last 0-15 bytes if any
+ blr
+
+
+// G4 Forward, long, 16-byte aligned, 32-byte cache, use DCBT but not DCBA.
+// r0/cr0 = #bytes to 32-byte align
+
+LForwardVecAlig32NoDcba:
+ bnel+ LAlign32 // align destination iff necessary
+ bl LPrepareForwardVectors
+ mtlr ra // restore return address before loading c128
+ li c128,128
+ b 1f // enter aligned loop
+
+ .align 4 // balance 13-word loop between QWs...
+ nop // ...which improves performance 5% +/-
+ nop
+1: // loop over aligned 64-byte chunks
+ dcbt c96,rs // pre-fetch three cache lines ahead
+ dcbt c128,rs // and four
+ lvx v1,0,rs
+ lvx v2,c16,rs
+ lvx v3,c32,rs
+ lvx v4,c48,rs
+ addi rs,rs,64
+ stvx v1,0,rd
+ stvx v2,c16,rd
+ stvx v3,c32,rd
+ stvx v4,c48,rd
+ addi rd,rd,64
+ bdnz 1b
+
+ b LForwardVectorAlignedEnd
+
+
+// G4 Forward, long, unaligned, 32-byte cache ops, use DCBT and DCBA. At least on
+// some CPUs, this routine is no slower than the simpler aligned version that does
+// not use permutes. But it cannot be used with aligned operands, because of the
+// way it prefetches source QWs.
+// r0/cr0 = #bytes to 32-byte align
+
+LForwardVecUnal32Dcba:
+ bnel+ LAlign32 // align destination iff necessary
+ bl LPrepareForwardVectors
+ lvx v1,0,rs // prime loop
+ mtlr ra // restore return address before loading c128
+ lvsl vp,0,rs // get permute vector to shift left
+ li c128,128
+ b 1f // enter aligned loop
+
+ .align 4 // long loop heads should be at least 16-byte aligned
+1: // loop over aligned 64-byte destination chunks
+ lvx v2,c16,rs
+ dcbt c96,rs // touch 3rd cache line ahead
+ lvx v3,c32,rs
+ dcbt c128,rs // touch 4th cache line ahead
+ lvx v4,c48,rs
+ addi rs,rs,64
+ vperm vx,v1,v2,vp
+ lvx v1,0,rs
+ vperm vy,v2,v3,vp
+ dcba 0,rd // avoid read of destination lines
+ stvx vx,0,rd
+ vperm vz,v3,v4,vp
+ stvx vy,c16,rd
+ dcba c32,rd
+ vperm vx,v4,v1,vp
+ stvx vz,c32,rd
+ stvx vx,c48,rd
+ addi rd,rd,64
+ bdnz 1b
+
+LForwardVectorUnalignedEnd: // r0/cr0=#QWs, rv=VRSave, v1=next QW, cr7=(rc & F), cr6 set on cr7
+ beq- 3f // no leftover quadwords
+ mtctr r0
+2: // loop over remaining quadwords
+ lvx v2,c16,rs
+ addi rs,rs,16
+ vperm vx,v1,v2,vp
+ vor v1,v2,v2 // v1 <- v2
+ stvx vx,0,rd
+ addi rd,rd,16
+ bdnz 2b
+3:
+ mtspr VRSave,rv // restore bitmap of live vr's
+ bne cr6,LShort16 // handle last 0-15 bytes if any
+ blr
+
+
+// G4 Forward, long, unaligned, 32-byte cache ops, use DCBT but not DCBA.
+// r0/cr0 = #bytes to 32-byte align
+
+LForwardVecUnal32NoDcba:
+ bnel+ LAlign32 // align destination iff necessary
+ bl LPrepareForwardVectors
+ lvx v1,0,rs // prime loop
+ mtlr ra // restore return address before loading c128
+ lvsl vp,0,rs // get permute vector to shift left
+ li c128,128
+ b 1f // enter aligned loop
+
+ .align 4
+ nop // balance 17-word loop between QWs
+ nop
+1: // loop over aligned 64-byte destination chunks
+ lvx v2,c16,rs
+ dcbt c96,rs // touch 3rd cache line ahead
+ lvx v3,c32,rs
+ dcbt c128,rs // touch 4th cache line ahead
+ lvx v4,c48,rs
+ addi rs,rs,64
+ vperm vx,v1,v2,vp
+ lvx v1,0,rs
+ vperm vy,v2,v3,vp
+ stvx vx,0,rd
+ vperm vz,v3,v4,vp
+ stvx vy,c16,rd
+ vperm vx,v4,v1,vp
+ stvx vz,c32,rd
+ stvx vx,c48,rd
+ addi rd,rd,64
+ bdnz 1b
+
+ b LForwardVectorUnalignedEnd
+
+
+// G3 Reverse, long, unaligned.
+
+LReverseWord:
+ bl LAlign8Reverse // 8-byte align destination
+ mtlr ra // restore return address
+ srwi r0,rc,5 // get count of 32-byte chunks to xfer (> 1)
+ mtctr r0
+1:
+ lwz w1,-4(rs)
+ lwz w2,-8(rs)
+ lwz w3,-12(rs)
+ lwz w4,-16(rs)
+ stw w1,-4(rd)
+ lwz w5,-20(rs)
+ stw w2,-8(rd)
+ lwz w6,-24(rs)
+ stw w3,-12(rd)
+ lwz w7,-28(rs)
+ stw w4,-16(rd)
+ lwzu w8,-32(rs)
+ stw w5,-20(rd)
+ stw w6,-24(rd)
+ stw w7,-28(rd)
+ stwu w8,-32(rd)
+ bdnz 1b
+
+ b LShortReverse
+
+
+// G3 Reverse, long, word aligned.
+
+LReverseFloat:
+ bl LAlign8Reverse // 8-byte align
+ mtlr ra // restore return address
+ srwi r0,rc,5 // get count of 32-byte chunks to xfer (> 1)
+ mtctr r0
+1:
+ lfd f0,-8(rs)
+ lfd f1,-16(rs)
+ lfd f2,-24(rs)
+ lfdu f3,-32(rs)
+ stfd f0,-8(rd)
+ stfd f1,-16(rd)
+ stfd f2,-24(rd)
+ stfdu f3,-32(rd)
+ bdnz 1b
+
+ b LShortReverse
+
+
+// G4 Reverse, long, 16-byte aligned, 32-byte DCBT but no DCBA.
+
+LReverseVectorAligned32:
+ bl LAlign32Reverse // 32-byte align destination iff necessary
+ bl LPrepareReverseVectors
+ mtlr ra // restore return address before loading cm129
+ li cm129,-129
+ b 1f // enter aligned loop
+
+ .align 4
+ nop // must start in 3rd word of QW...
+ nop // ...to keep balanced
+1: // loop over aligned 64-byte chunks
+ dcbt cm97,rs // pre-fetch three cache lines ahead
+ dcbt cm129,rs // and four
+ lvx v1,cm1,rs
+ lvx v2,cm17,rs
+ lvx v3,cm33,rs
+ lvx v4,cm49,rs
+ subi rs,rs,64
+ stvx v1,cm1,rd
+ stvx v2,cm17,rd
+ stvx v3,cm33,rd
+ stvx v4,cm49,rd
+ subi rd,rd,64
+ bdnz 1b
+
+LReverseVectorAlignedEnd: // cr0/r0=#quadwords, rv=VRSave, cr7=low 4 bits of rc, cr6 set on cr7
+ beq 3f // no leftover quadwords
+ mtctr r0
+2: // loop over 1-3 quadwords
+ lvx v1,cm1,rs
+ subi rs,rs,16
+ stvx v1,cm1,rd
+ subi rd,rd,16
+ bdnz 2b
+3:
+ mtspr VRSave,rv // restore bitmap of live vr's
+ bne cr6,LShortReverse16 // handle last 0-15 bytes iff any
+ blr
+
+
+// G4 Reverse, long, unaligned, 32-byte DCBT.
+
+LReverseVectorUnal32:
+ bl LAlign32Reverse // align destination iff necessary
+ bl LPrepareReverseVectors
+ lvx v1,cm1,rs // prime loop
+ mtlr ra // restore return address before loading cm129
+ lvsl vp,0,rs // get permute vector to shift left
+ li cm129,-129
+ b 1f // enter aligned loop
+
+ .align 4
+ nop // start loop in 3rd word on QW to balance
+ nop
+1: // loop over aligned 64-byte destination chunks
+ lvx v2,cm17,rs
+ dcbt cm97,rs // touch in 3rd source block
+ lvx v3,cm33,rs
+ dcbt cm129,rs // touch in 4th
+ lvx v4,cm49,rs
+ subi rs,rs,64
+ vperm vx,v2,v1,vp
+ lvx v1,cm1,rs
+ vperm vy,v3,v2,vp
+ stvx vx,cm1,rd
+ vperm vz,v4,v3,vp
+ stvx vy,cm17,rd
+ vperm vx,v1,v4,vp
+ stvx vz,cm33,rd
+ stvx vx,cm49,rd
+ subi rd,rd,64
+ bdnz 1b
+
+LReverseVectorUnalignedEnd: // r0/cr0=#QWs, rv=VRSave, v1=source QW, cr7=low 4 bits of rc, cr6 set on cr7
+ beq 3f // no leftover quadwords
+ mtctr r0
+2: // loop over 1-3 quadwords
+ lvx v2,cm17,rs
+ subi rs,rs,16
+ vperm vx,v2,v1,vp
+ vor v1,v2,v2 // v1 <- v2
+ stvx vx,cm1,rd
+ subi rd,rd,16
+ bdnz 2b
+3:
+ mtspr VRSave,rv // restore bitmap of live vr's
+ bne cr6,LShortReverse16 // handle last 0-15 bytes iff any
+ blr
+
+
+// Subroutine to prepare for 64-byte forward vector loops.
+// Returns many things:
+// ctr = number of 64-byte chunks to move
+// r0/cr0 = leftover QWs to move
+// cr7 = low 4 bits of rc (ie, leftover byte count 0-15)
+// cr6 = beq if leftover byte count is 0
+// c16..c96 loaded
+// rv = original value of VRSave
+// NB: c128 not set (if needed), since it is still "ra"
+
+LPrepareForwardVectors:
+ mfspr rv,VRSave // get bitmap of live vector registers
+ srwi r0,rc,6 // get count of 64-byte chunks to move (>=1)
+ oris w1,rv,0xFF00 // we use v0-v7
+ mtcrf 0x01,rc // prepare for moving last 0-15 bytes in LShort16
+ rlwinm w3,rc,0,28,31 // move last 0-15 byte count to w3 too
+ mtspr VRSave,w1 // update mask
+ li c16,16 // get constants used in ldvx/stvx
+ li c32,32
+ mtctr r0 // set up loop count
+ cmpwi cr6,w3,0 // set cr6 on leftover byte count
+ li c48,48
+ li c96,96
+ rlwinm. r0,rc,28,30,31 // get number of quadword leftovers (0-3) and set cr0
+ blr
+
+
+// Subroutine to prepare for 64-byte reverse vector loops.
+// Returns many things:
+// ctr = number of 64-byte chunks to move
+// r0/cr0 = leftover QWs to move
+// cr7 = low 4 bits of rc (ie, leftover byte count 0-15)
+// cr6 = beq if leftover byte count is 0
+// cm1..cm97 loaded
+// rv = original value of VRSave
+// NB: cm129 not set (if needed), since it is still "ra"
+
+LPrepareReverseVectors:
+ mfspr rv,VRSave // get bitmap of live vector registers
+ srwi r0,rc,6 // get count of 64-byte chunks to move (>=1)
+ oris w1,rv,0xFF00 // we use v0-v7
+ mtcrf 0x01,rc // prepare for moving last 0-15 bytes in LShortReverse16
+ rlwinm w3,rc,0,28,31 // move last 0-15 byte count to w3 too
+ mtspr VRSave,w1 // update mask
+ li cm1,-1 // get constants used in ldvx/stvx
+ li cm17,-17
+ mtctr r0 // set up loop count
+ cmpwi cr6,w3,0 // set cr6 on leftover byte count
+ li cm33,-33
+ li cm49,-49
+ rlwinm. r0,rc,28,30,31 // get number of quadword leftovers (0-3) and set cr0
+ li cm97,-97
+ blr
+
+
+// Subroutine to align destination on a 32-byte boundary.
+// r0 = number of bytes to xfer (0-31)
+
+LAlign32:
+ mtcrf 0x01,r0 // length to cr (faster to change 1 CR at a time)
+ mtcrf 0x02,r0
+ sub rc,rc,r0 // adjust length
+ bf 31,1f // skip if no odd bit
+ lbz w1,0(rs)
+ addi rs,rs,1
+ stb w1,0(rd)
+ addi rd,rd,1
+1:
+ bf 30,2f // halfword to move?
+ lhz w1,0(rs)
+ addi rs,rs,2
+ sth w1,0(rd)
+ addi rd,rd,2
+2:
+ bf 29,3f // word?
+ lwz w1,0(rs)
+ addi rs,rs,4
+ stw w1,0(rd)
+ addi rd,rd,4
+3:
+ bf 28,4f // doubleword?
+ lwz w1,0(rs)
+ lwz w2,4(rs)
+ addi rs,rs,8
+ stw w1,0(rd)
+ stw w2,4(rd)
+ addi rd,rd,8
+4:
+ bflr 27 // done if no quadword to move
+ lwz w1,0(rs)
+ lwz w2,4(rs)
+ lwz w3,8(rs)
+ lwz w4,12(rs)
+ addi rs,rs,16
+ stw w1,0(rd)
+ stw w2,4(rd)
+ stw w3,8(rd)
+ stw w4,12(rd)
+ addi rd,rd,16
+ blr
+
+// Subroutine to align destination if necessary on a 32-byte boundary for reverse moves.
+// rs and rd still point to low end of operands
+// we adjust rs and rd to point to last byte moved
+
+LAlign32Reverse:
+ add rd,rd,rc // point to last byte moved (ie, 1 past end of operands)
+ add rs,rs,rc
+ andi. r0,rd,0x1F // r0 <- #bytes that must be moved to align destination
+ mtcrf 0x01,r0 // length to cr (faster to change 1 CR at a time)
+ mtcrf 0x02,r0
+ sub rc,rc,r0 // update length
+ beqlr- // destination already 32-byte aligned
+
+ bf 31,1f // odd byte?
+ lbzu w1,-1(rs)
+ stbu w1,-1(rd)
+1:
+ bf 30,2f // halfword to move?
+ lhzu w1,-2(rs)
+ sthu w1,-2(rd)
+2:
+ bf 29,3f // word?
+ lwzu w1,-4(rs)
+ stwu w1,-4(rd)
+3:
+ bf 28,4f // doubleword?
+ lwz w1,-4(rs)
+ lwzu w2,-8(rs)
+ stw w1,-4(rd)
+ stwu w2,-8(rd
+4:
+ bflr 27 // done if no quadwords
+ lwz w1,-4(rs)
+ lwz w2,-8(rs)
+ lwz w3,-12(rs)
+ lwzu w4,-16(rs)
+ stw w1,-4(rd)
+ stw w2,-8(rd)
+ stw w3,-12(rd)
+ stwu w4,-16(rd)
+ blr
+
+
+// Subroutine to align destination on an 8-byte boundary for reverse moves.
+// rs and rd still point to low end of operands
+// we adjust rs and rd to point to last byte moved
+
+LAlign8Reverse:
+ add rd,rd,rc // point to last byte moved (ie, 1 past end of operands)
+ add rs,rs,rc
+ andi. r0,rd,0x7 // r0 <- #bytes that must be moved to align destination
+ beqlr- // destination already 8-byte aligned
+ mtctr r0 // set up for loop
+ sub rc,rc,r0 // update length
+1:
+ lbzu w1,-1(rs)
+ stbu w1,-1(rd)
+ bdnz 1b
+
+ blr
+
+
+// Called by pthread initialization to set up the branch table pointer based on
+// the CPU capability vector. This routine may be called more than once (for
+// example, during testing.)
+
+// Size of the buffer we use to do DCBA timing on G4:
+#define kBufSiz 1024
+
+// Stack frame size, which contains the 128-byte-aligned buffer:
+#define kSFSize (kBufSiz+128+16)
+
+// Iterations of the timing loop:
+#define kLoopCnt 5
+
+// Bit in cr5 used as a flag in timing loop:
+#define kDCBA 22
+
+__bcopy_initialize: // int _bcopy_initialize(void)
+ mflr ra // get return
+ stw ra,8(r1) // save
+ stwu r1,-kSFSize(r1) // carve our temp buffer from the stack
+ addi w6,r1,127+16 // get base address...
+ rlwinm w6,w6,0,0,24 // ...of our buffer, 128-byte aligned
+ bcl 20,31,1f // get our PIC base
+1:
+ mflr w1
+ addis w2,w1,ha16(__cpu_capabilities - 1b)
+ lwz w3,lo16(__cpu_capabilities - 1b)(w2)
+ andi. r0,w3,kUseDcba+kNoDcba+kCache32+k64Bit+kHasAltivec
+ cmpwi r0,kCache32+kHasAltivec // untyped G4?
+ li w8,0 // assume no need to test
+ bne 2f // not an untyped G4, so do not test
+
+ // G4, but neither kUseDcba or kNoDcba are set. Time and select fastest.
+
+ crset kDCBA // first, use DCBA
+ bl LTest32 // time it
+ mr w8,w4 // w8 <- best time using DCBA
+ srwi r0,w8,3 // bias 12 pct in favor of not using DCBA...
+ add w8,w8,r0 // ...because DCBA is always slower with warm cache
+ crclr kDCBA
+ bl LTest32 // w4 <- best time without DCBA
+ cmplw w8,w4 // which is better?
+ li w8,kUseDcba // assume using DCBA is faster
+ blt 2f
+ li w8,kNoDcba // no DCBA is faster
+
+ // What branch table to use?
+
+2: // here with w8 = 0, kUseDcba, or kNoDcba
+ bcl 20,31,4f // get our PIC base again
+4:
+ mflr w1
+ addis w2,w1,ha16(__cpu_capabilities - 4b)
+ lwz w3,lo16(__cpu_capabilities - 4b)(w2)
+ or w3,w3,w8 // add in kUseDcba or kNoDcba if untyped G4
+ mr r3,w8 // return dynamic selection, if any (used in testing)
+
+ andi. r0,w3,kHasAltivec+k64Bit+kCache128+kCache64+kCache32+kUseDcba+kNoDcba
+ cmpwi r0,kHasAltivec+kCache32+kUseDcba // G4 with DCBA?
+ addis w4,w1,ha16(LG4UseDcba - 4b)
+ addi w4,w4,lo16(LG4UseDcba - 4b)
+ beq 5f
+
+ andi. r0,w3,kHasAltivec+k64Bit+kCache128+kCache64+kCache32+kUseDcba+kNoDcba
+ cmpwi r0,kHasAltivec+kCache32+kNoDcba // G4 without DCBA?
+ addis w4,w1,ha16(LG4NoDcba - 4b)
+ addi w4,w4,lo16(LG4NoDcba - 4b)
+ beq 5f
+
+ andi. r0,w3,kHasAltivec+k64Bit+kCache128+kCache64+kCache32
+ cmpwi r0,kCache32 // G3?
+ addis w4,w1,ha16(LG3 - 4b)
+ addi w4,w4,lo16(LG3 - 4b)
+ beq 5f
+
+ // Map unrecognized CPU types to G3 (lowest common denominator)
+
+5: // w4 <- branch table pointer
+ addis w5,w1,ha16(LBranchTablePtr - 4b)
+ stw w4,lo16(LBranchTablePtr - 4b)(w5)
+ lwz ra,kSFSize+8(r1) // recover return address
+ mtlr ra // restore it
+ lwz r1,0(r1) // pop off our stack frame
+ blr // return dynamic selection (or 0) in r3
+
+
+// Subroutine to time a 32-byte cache.
+// kDCBA = set if we should use DCBA
+// w6 = base of buffer to use for test (kBufSiz bytes)
+// w4 = we return time of fastest loop in w4
+
+LTest32:
+ li w1,kLoopCnt // number of times to loop
+ li w4,-1 // initialize fastest time
+1:
+ mr rd,w6 // initialize buffer ptr
+ li r0,kBufSiz/32 // r0 <- cache blocks to test
+ mtctr r0
+2:
+ dcbf 0,rd // first, force the blocks out of the cache
+ addi rd,rd,32
+ bdnz 2b
+ sync // make sure all the flushes take
+ mr rd,w6 // re-initialize buffer ptr
+ mtctr r0 // reset cache-block count
+ mftbu w5 // remember upper half so we can check for carry
+ mftb w2 // start the timer
+3: // loop over cache blocks
+ bf kDCBA,4f // should we DCBA?
+ dcba 0,rd
+4:
+ stfd f1,0(rd) // store the entire cache block
+ stfd f1,8(rd)
+ stfd f1,16(rd)
+ stfd f1,24(rd)
+ addi rd,rd,32
+ bdnz 3b
+ mftb w3
+ mftbu r0
+ cmpw r0,w5 // did timebase carry?
+ bne 1b // yes, retest rather than fuss
+ sub w3,w3,w2 // w3 <- time for this loop
+ cmplw w3,w4 // faster than current best?
+ bge 5f // no
+ mr w4,w3 // remember fastest time through loop
+5:
+ subi w1,w1,1 // decrement outer loop count
+ cmpwi w1,0 // more to go?
+ bne 1b // loop if so
+ blr
+