Loading...
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 | /* * Copyright (c) 2003 Apple Computer, Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * * This file contains Original Code and/or Modifications of Original Code * as defined in and that are subject to the Apple Public Source License * Version 2.0 (the 'License'). You may not use this file except in * compliance with the License. The rights granted to you under the License * may not be used to create, or enable the creation or redistribution of, * unlawful or unlicensed copies of an Apple operating system, or to * circumvent, violate, or enable the circumvention or violation of, any * terms of an Apple operating system software license agreement. * * Please obtain a copy of the License at * http://www.opensource.apple.com/apsl/ and read it before using this file. * * The Original Code and all software distributed under the License are * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. * Please see the License for the specific language governing rights and * limitations under the License. * * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ /* ======================================= * BCOPY, MEMCPY, and MEMMOVE for Mac OS X * ======================================= * * Version of 2/20/2003, tuned for G4. The inner loops use DCBA to avoid * reading destination cache lines. Only the 7450 actually benefits from * this, and then only in the cold-cache case. On 7400s and 7455s, we * patch the DCBAs into NOPs. * * Register usage. Note we use R2, so this code will not run in a PEF/CFM * environment. Note also the rather delicate way we assign multiple uses * to the same register. Beware. * * r0 = "w7" or temp (NB: cannot use r0 for any constant such as "c16") * r2 = "w8" or vrsave ("rv") * r3 = not used, as memcpy and memmove return 1st parameter as a value * r4 = source ptr ("rs") * r5 = count of bytes to move ("rc") * r6 = "w1", "c16", or "cm17" * r7 = "w2", "c32", or "cm33" * r8 = "w3", "c48", or "cm49" * r9 = "w4", or "cm1" * r10 = "w5", "c96", or "cm97" * r11 = "w6", "c128", or "cm129" * r12 = destination ptr ("rd") * v0 = permute vector ("vp") * v1-v4 = qw's loaded from source * v5-v7 = permuted qw's ("vw", "vx", "vy") */ #define rs r4 #define rd r12 #define rc r5 #define rv r2 #define w1 r6 #define w2 r7 #define w3 r8 #define w4 r9 #define w5 r10 #define w6 r11 #define w7 r0 #define w8 r2 #define c16 r6 #define cm17 r6 #define c32 r7 #define cm33 r7 #define c48 r8 #define cm49 r8 #define cm1 r9 #define c96 r10 #define cm97 r10 #define c128 r11 #define cm129 r11 #define vp v0 #define vw v5 #define vx v6 #define vy v7 #include <sys/appleapiopts.h> #include <ppc/asm.h> #include <machine/cpu_capabilities.h> #include <machine/commpage.h> .text #define kMedium 32 // too long for inline loopless code #define kLong 96 // long enough to justify use of Altivec // Main entry points. .align 5 bcopy_g4: // void bcopy(const void *src, void *dst, size_t len) cmplwi rc,kMedium // short or long? sub w1,r4,r3 // must move in reverse if (rd-rs)<rc cmplw cr1,w1,rc // set cr1 blt iff we must move reverse mr rd,r4 // start to move registers to canonic spot mr rs,r3 blt+ LShort // handle short operands dcbt 0,r3 // touch in destination b LMedium // join medium/long operand code // NB: memmove() must be 8 words past bcopy(), to agree with comm page addresses. .align 5 Lmemcpy_g4: // void* memcpy(void *dst, void *src, size_t len) Lmemmove_g4: // void* memmove(void *dst, const void *src, size_t len) cmplwi rc,kMedium // short or long? sub w1,r3,r4 // must move in reverse if (rd-rs)<rc dcbt 0,r4 // touch in the first line of source cmplw cr1,w1,rc // set cr1 blt iff we must move reverse mr rd,r3 // must leave r3 alone, it is return value for memcpy etc bge- LMedium // handle medium or long operands // Handle short operands. LShort: andi. r0,rc,0x10 // test bit 27 separately (faster on G4) mtcrf 0x01,rc // put length bits 28-31 in cr7 blt- cr1,LShortReverse // Forward short operands. This is the most frequent case, so it is inline. beq LShort16 // quadword to move? lwz w1,0(rs) lwz w2,4(rs) lwz w3,8(rs) lwz w4,12(rs) addi rs,rs,16 stw w1,0(rd) stw w2,4(rd) stw w3,8(rd) stw w4,12(rd) addi rd,rd,16 LShort16: // join here to xfer 0-15 bytes bf 28,2f // doubleword? lwz w1,0(rs) lwz w2,4(rs) addi rs,rs,8 stw w1,0(rd) stw w2,4(rd) addi rd,rd,8 2: bf 29,3f // word? lwz w1,0(rs) addi rs,rs,4 stw w1,0(rd) addi rd,rd,4 3: bf 30,4f // halfword to move? lhz w1,0(rs) addi rs,rs,2 sth w1,0(rd) addi rd,rd,2 4: bflr 31 // skip if no odd byte lbz w1,0(rs) stb w1,0(rd) blr // Handle short reverse operands. // cr0 = bne if bit 27 of length is set // cr7 = bits 28-31 of length LShortReverse: add rs,rs,rc // adjust ptrs for reverse move add rd,rd,rc beq LShortReverse16 // quadword to move? lwz w1,-4(rs) lwz w2,-8(rs) lwz w3,-12(rs) lwzu w4,-16(rs) stw w1,-4(rd) stw w2,-8(rd) stw w3,-12(rd) stwu w4,-16(rd) LShortReverse16: // join here to xfer 0-15 bytes and return bf 28,2f // doubleword? lwz w1,-4(rs) lwzu w2,-8(rs) stw w1,-4(rd) stwu w2,-8(rd) 2: bf 29,3f // word? lwzu w1,-4(rs) stwu w1,-4(rd) 3: bf 30,4f // halfword to move? lhzu w1,-2(rs) sthu w1,-2(rd) 4: bflr 31 // done if no odd byte lbz w1,-1(rs) // no update stb w1,-1(rd) blr // Medium and long operands. Use Altivec if long enough, else scalar loops. // w1 = (rd-rs), used to check for alignment // cr1 = blt iff we must move reverse .align 4 LMedium: dcbtst 0,rd // touch in destination cmplwi cr7,rc,kLong // long enough for vectors? neg w3,rd // start to compute #bytes to align destination rlwinm r0,w1,0,0x7 // check relative 8-byte alignment andi. w6,w3,7 // w6 <- #bytes to 8-byte align destination blt cr1,LMediumReverse // handle reverse moves rlwinm w4,w3,0,0x1F // w4 <- #bytes to 32-byte align destination cmpwi cr6,r0,0 // set cr6 beq if relatively aligned bge cr7,LFwdLong // long enough for vectors // Medium length: use scalar loops. // w6/cr0 = #bytes to 8-byte align destination // cr6 = beq if relatively doubleword aligned sub rc,rc,w6 // decrement length remaining beq 1f // skip if dest already doubleword aligned mtxer w6 // set up count for move lswx w1,0,rs // move w6 bytes to align destination stswx w1,0,rd add rs,rs,w6 // bump ptrs past add rd,rd,w6 1: srwi r0,rc,4 // get # 16-byte chunks (>=1) mtcrf 0x01,rc // save remaining byte count here for LShort16 mtctr r0 // set up 16-byte loop bne cr6,3f // source not 4-byte aligned b 2f .align 4 2: // loop over 16-byte aligned chunks lfd f0,0(rs) lfd f1,8(rs) addi rs,rs,16 stfd f0,0(rd) stfd f1,8(rd) addi rd,rd,16 bdnz 2b b LShort16 .align 4 3: // loop over 16-byte unaligned chunks lwz w1,0(rs) lwz w2,4(rs) lwz w3,8(rs) lwz w4,12(rs) addi rs,rs,16 stw w1,0(rd) stw w2,4(rd) stw w3,8(rd) stw w4,12(rd) addi rd,rd,16 bdnz 3b b LShort16 // Vector loops. First, we must 32-byte align the destination. // w1 = (rd-rs), used to check for reverse and alignment // w4 = #bytes to 32-byte align destination // rc = long enough for at least one vector loop LFwdLong: cmpwi w4,0 // dest already aligned? sub rc,rc,w4 // adjust length mtcrf 0x01,w4 // cr7 <- #bytes to align dest rlwinm w2,w1,0,0xF // relatively 16-byte aligned? mtcrf 0x02,w4 // finish moving #bytes to align to cr6 and cr7 srwi r0,rc,6 // get # 64-byte chunks to xfer (>=1) cmpwi cr5,w2,0 // set cr5 beq if relatively 16-byte aligned beq LFwdAligned // dest is already aligned // 32-byte align destination. bf 31,1f // byte to move? lbz w1,0(rs) addi rs,rs,1 stb w1,0(rd) addi rd,rd,1 1: bf 30,2f // halfword? lhz w1,0(rs) addi rs,rs,2 sth w1,0(rd) addi rd,rd,2 2: bf 29,3f // word? lwz w1,0(rs) addi rs,rs,4 stw w1,0(rd) addi rd,rd,4 3: bf 28,4f // doubleword? lwz w1,0(rs) lwz w2,4(rs) addi rs,rs,8 stw w1,0(rd) stw w2,4(rd) addi rd,rd,8 4: bf 27,LFwdAligned // quadword? lwz w1,0(rs) lwz w2,4(rs) lwz w3,8(rs) lwz w4,12(rs) addi rs,rs,16 stw w1,0(rd) stw w2,4(rd) stw w3,8(rd) stw w4,12(rd) addi rd,rd,16 // Destination is 32-byte aligned. // r0 = count of 64-byte chunks to move (not 0) // rd = 32-byte aligned // rc = bytes remaining // cr5 = beq if source is 16-byte aligned // We set up many registers: // ctr = number of 64-byte chunks to move // r0/cr0 = leftover QWs to move // cr7 = low 4 bits of rc (ie, leftover byte count 0-15) // cr6 = beq if leftover byte count is 0 // rv = original value of vrsave // c16 etc = loaded LFwdAligned: mfspr rv,vrsave // get bitmap of live vector registers mtcrf 0x01,rc // move leftover count to cr7 for LShort16 rlwinm w3,rc,0,28,31 // move last 0-15 byte count to w3 mtctr r0 // set up loop count cmpwi cr6,w3,0 // set cr6 on leftover byte count oris w1,rv,0xFF00 // we use v0-v7 rlwinm. r0,rc,28,30,31 // get number of quadword leftovers (0-3) and set cr0 mtspr vrsave,w1 // update mask li c16,16 // get constants used in ldvx/stvx li c32,32 li c48,48 li c96,96 li c128,128 bne cr5,LForwardVecUnal // handle unaligned operands b 1f .align 4 1: // loop over 64-byte chunks dcbt c96,rs dcbt c128,rs lvx v1,0,rs lvx v2,c16,rs lvx v3,c32,rs lvx v4,c48,rs addi rs,rs,64 dcba 0,rd // patched to NOP on some machines stvx v1,0,rd stvx v2,c16,rd dcba c32,rd // patched to NOP on some machines stvx v3,c32,rd stvx v4,c48,rd addi rd,rd,64 bdnz 1b beq 4f // no leftover quadwords mtctr r0 3: // loop over remaining quadwords (1-3) lvx v1,0,rs addi rs,rs,16 stvx v1,0,rd addi rd,rd,16 bdnz 3b 4: mtspr vrsave,rv // restore bitmap of live vr's bne cr6,LShort16 // handle last 0-15 bytes if any blr // Long, forward, unaligned vector loop. LForwardVecUnal: lvsl vp,0,rs // get permute vector to shift left lvx v1,0,rs // prefetch 1st source quadword b 1f .align 4 // align inner loops 1: // loop over 64-byte chunks lvx v2,c16,rs dcbt c96,rs lvx v3,c32,rs dcbt c128,rs lvx v4,c48,rs addi rs,rs,64 vperm vw,v1,v2,vp lvx v1,0,rs vperm vx,v2,v3,vp dcba 0,rd // patched to NOP on some machines stvx vw,0,rd vperm vy,v3,v4,vp stvx vx,c16,rd vperm vw,v4,v1,vp dcba c32,rd // patched to NOP on some machines stvx vy,c32,rd stvx vw,c48,rd addi rd,rd,64 bdnz 1b beq- 4f // no leftover quadwords mtctr r0 3: // loop over remaining quadwords lvx v2,c16,rs addi rs,rs,16 vperm vx,v1,v2,vp vor v1,v2,v2 // v1 <- v2 stvx vx,0,rd addi rd,rd,16 bdnz 3b 4: mtspr vrsave,rv // restore bitmap of live vr's bne cr6,LShort16 // handle last 0-15 bytes if any blr // Medium and long, reverse moves. We use altivec if the operands are long enough, // else a lwz/stx loop. // w1 = (rd-rs), used to check for reverse and alignment // cr7 = bge if long LMediumReverse: add rd,rd,rc // point to end of operands add rs,rs,rc andi. w4,rd,0x1F // w4 <- #bytes to 32-byte align destination rlwinm w6,rd,0,0x3 // w6 <- #bytes to 4-byte align destination bge cr7,LLongReverse // long enough for vectors // Scalar loop. // w6 = #bytes to 4-byte align destination sub rc,rc,w6 // decrement length remaining mtxer w6 // set up count for move sub rs,rs,w6 // back up ptrs sub rd,rd,w6 srwi r0,rc,4 // get # 16-byte chunks (>=1) mtcrf 0x01,rc // set remaining byte count here for LShortReverse16 lswx w1,0,rs // move w6 bytes to align destination stswx w1,0,rd mtctr r0 // set up 16-byte loop b 1f .align 4 1: // loop over 16-byte aligned chunks lwz w1,-4(rs) lwz w2,-8(rs) lwz w3,-12(rs) lwzu w4,-16(rs) stw w1,-4(rd) stw w2,-8(rd) stw w3,-12(rd) stwu w4,-16(rd) bdnz 1b b LShortReverse16 // Reverse vector loops. First, we must 32-byte align the destination. // w1 = (rd-rs), used to check for reverse and alignment // w4/cr0 = #bytes to 32-byte align destination // rc = long enough for at least one vector loop LLongReverse: sub rc,rc,w4 // adjust length mtcrf 0x01,w4 // cr7 <- #bytes to align dest rlwinm w2,w1,0,0xF // relatively 16-byte aligned? mtcrf 0x02,w4 // finish moving #bytes to align to cr6 and cr7 srwi r0,rc,6 // get # 64-byte chunks to xfer (>=1) cmpwi cr5,w2,0 // set cr5 beq if relatively 16-byte aligned beq LReverseAligned // dest is already aligned // 32-byte align destination. bf 31,1f // byte to move? lbzu w1,-1(rs) stbu w1,-1(rd) 1: bf 30,2f // halfword? lhzu w1,-2(rs) sthu w1,-2(rd) 2: bf 29,3f // word? lwzu w1,-4(rs) stwu w1,-4(rd) 3: bf 28,4f // doubleword? lwz w1,-4(rs) lwzu w2,-8(rs) stw w1,-4(rd) stwu w2,-8(rd) 4: bf 27,LReverseAligned // quadword? lwz w1,-4(rs) lwz w2,-8(rs) lwz w3,-12(rs) lwzu w4,-16(rs) stw w1,-4(rd) stw w2,-8(rd) stw w3,-12(rd) stwu w4,-16(rd) // Destination is 32-byte aligned. // r0 = count of 64-byte chunks to move (not 0) // rd = 32-byte aligned // rc = bytes remaining // cr5 = beq if source is 16-byte aligned // We set up many registers: // ctr = number of 64-byte chunks to move // r0/cr0 = leftover QWs to move // cr7 = low 4 bits of rc (ie, leftover byte count 0-15) // cr6 = beq if leftover byte count is 0 // rv = original value of vrsave // cm1 etc = loaded LReverseAligned: mfspr rv,vrsave // get bitmap of live vector registers mtcrf 0x01,rc // move leftover count to cr7 for LShort16 rlwinm w3,rc,0,28,31 // move last 0-15 byte count to w3 mtctr r0 // set up loop count cmpwi cr6,w3,0 // set cr6 on leftover byte count oris w1,rv,0xFF00 // we use v0-v7 rlwinm. r0,rc,28,30,31 // get number of quadword leftovers (0-3) and set cr0 mtspr vrsave,w1 // update mask li cm1,-1 // get constants used in ldvx/stvx li cm17,-17 li cm33,-33 li cm49,-49 li cm97,-97 li cm129,-129 bne cr5,LReverseVecUnal // handle unaligned operands b 1f .align 4 // align inner loops 1: // loop over 64-byte chunks dcbt cm97,rs dcbt cm129,rs lvx v1,cm1,rs lvx v2,cm17,rs lvx v3,cm33,rs lvx v4,cm49,rs subi rs,rs,64 stvx v1,cm1,rd stvx v2,cm17,rd stvx v3,cm33,rd stvx v4,cm49,rd subi rd,rd,64 bdnz 1b beq 4f // no leftover quadwords mtctr r0 3: // loop over remaining quadwords (1-7) lvx v1,cm1,rs subi rs,rs,16 stvx v1,cm1,rd subi rd,rd,16 bdnz 3b 4: mtspr vrsave,rv // restore bitmap of live vr's bne cr6,LShortReverse16 // handle last 0-15 bytes if any blr // Long, reverse, unaligned vector loop. LReverseVecUnal: lvsl vp,0,rs // get permute vector to shift left lvx v1,cm1,rs // v1 always looks ahead b 1f .align 4 // align the inner loops 1: // loop over 64-byte chunks lvx v2,cm17,rs dcbt cm97,rs lvx v3,cm33,rs dcbt cm129,rs lvx v4,cm49,rs subi rs,rs,64 vperm vw,v2,v1,vp lvx v1,cm1,rs vperm vx,v3,v2,vp stvx vw,cm1,rd vperm vy,v4,v3,vp stvx vx,cm17,rd vperm vw,v1,v4,vp stvx vy,cm33,rd stvx vw,cm49,rd subi rd,rd,64 bdnz 1b beq 3f // no leftover quadwords mtctr r0 2: // loop over 1-3 quadwords lvx v2,cm17,rs subi rs,rs,16 vperm vx,v2,v1,vp vor v1,v2,v2 // v1 <- v2 stvx vx,cm1,rd subi rd,rd,16 bdnz 2b 3: mtspr vrsave,rv // restore bitmap of live vr's bne cr6,LShortReverse16 // handle last 0-15 bytes iff any blr COMMPAGE_DESCRIPTOR(bcopy_g4,_COMM_PAGE_BCOPY,kHasAltivec,k64Bit,kCommPageDCBA+kCommPage32) |