path: root/arch/sparc/lib/M7memcpy.S

/*
 * M7memcpy: Optimized SPARC M7 memcpy
 *
 * Copyright (c) 2016, Oracle and/or its affiliates. All rights reserved.
 */

	.file	"M7memcpy.S"

/*
 * memcpy(s1, s2, len)
 *
 * Copy s2 to s1, always copy n bytes.
 * Note: this C code does not work for overlapped copies.
 *
 * Fast assembler language version of the following C-program for memcpy
 * which represents the `standard' for the C-library.
 *
 *	void *
 *	memcpy(void *s, const void *s0, size_t n)
 *	{
 *		if (n != 0) {
 *		    char *s1 = s;
 *		    const char *s2 = s0;
 *		    do {
 *			*s1++ = *s2++;
 *		    } while (--n != 0);
 *		}
 *		return (s);
 *	}
 *
 *
 * SPARC T7/M7 Flow :
 *
 * if (count < SMALL_MAX) {
 *   if count < SHORTCOPY              (SHORTCOPY=3)
 *	copy bytes; exit with dst addr
 *   if src & dst aligned on word boundary but not long word boundary,
 *     copy with ldw/stw; branch to finish_up
 *   if src & dst aligned on long word boundary
 *     copy with ldx/stx; branch to finish_up
 *   if src & dst not aligned and length <= SHORTCHECK   (SHORTCHECK=14)
 *     copy bytes; exit with dst addr
 *   move enough bytes to get src to word boundary
 *   if dst now on word boundary
 * move_words:
 *     copy words; branch to finish_up
 *   if dst now on half word boundary
 *     load words, shift half words, store words; branch to finish_up
 *   if dst on byte 1
 *     load words, shift 3 bytes, store words; branch to finish_up
 *   if dst on byte 3
 *     load words, shift 1 byte, store words; branch to finish_up
 * finish_up:
 *     copy bytes; exit with dst addr
 * } else {                                         More than SMALL_MAX bytes
 *   move bytes until dst is on long word boundary
 *   if( src is on long word boundary ) {
 *     if (count < MED_MAX) {
 * finish_long:					   src/dst aligned on 8 bytes
 *       copy with ldx/stx in 8-way unrolled loop;
 *       copy final 0-63 bytes; exit with dst addr
 *     } else {				     src/dst aligned; count > MED_MAX
 *       align dst on 64 byte boundary; for main data movement:
 *       prefetch src data to L2 cache; let HW prefetch move data to L1 cache
 *       Use BIS (block initializing store) to avoid copying store cache
 *       lines from memory. But pre-store first element of each cache line
 *       ST_CHUNK lines in advance of the rest of that cache line. That
 *       gives time for replacement cache lines to be written back without
 *       excess STQ and Miss Buffer filling. Repeat until near the end,
 *       then finish up storing before going to finish_long.
 *     }
 *   } else {                                   src/dst not aligned on 8 bytes
 *     if src is word aligned and count < MED_WMAX
 *       move words in 8-way unrolled loop
 *       move final 0-31 bytes; exit with dst addr
 *     if count < MED_UMAX
 *       use alignaddr/faligndata combined with ldd/std in 8-way
 *       unrolled loop to move data.
 *       go to unalign_done
 *     else
 *       setup alignaddr for faligndata instructions
 *       align dst on 64 byte boundary; prefetch src data to L1 cache
 *       loadx8, falign, block-store, prefetch loop
 *	 (only use block-init-store when src/dst on 8 byte boundaries.)
 * unalign_done:
 *       move remaining bytes for unaligned cases. exit with dst addr.
 * }
 *
 */

#include <asm/visasm.h>
#include <asm/asi.h>

#if !defined(EX_LD) && !defined(EX_ST)
#define NON_USER_COPY
#endif

#ifndef EX_LD
#define EX_LD(x,y)	x
#endif
#ifndef EX_LD_FP
#define EX_LD_FP(x,y)	x
#endif

#ifndef EX_ST
#define EX_ST(x,y)	x
#endif
#ifndef EX_ST_FP
#define EX_ST_FP(x,y)	x
#endif

#ifndef EX_RETVAL
#define EX_RETVAL(x)    x
#endif

#ifndef LOAD
#define LOAD(type,addr,dest)	type [addr], dest
#endif

#ifndef STORE
#define STORE(type,src,addr)	type src, [addr]
#endif

/*
 * ASI_BLK_INIT_QUAD_LDD_P/ASI_BLK_INIT_QUAD_LDD_S marks the cache
 * line as "least recently used" which means if many threads are
 * active, it has a high probability of being pushed out of the cache
 * between the first initializing store and the final stores.
 * Thus, we use ASI_ST_BLKINIT_MRU_P/ASI_ST_BLKINIT_MRU_S which
 * marks the cache line as "most recently used" for all
 * but the last cache line
 */
#ifndef STORE_ASI
#ifndef SIMULATE_NIAGARA_ON_NON_NIAGARA
#define STORE_ASI	ASI_BLK_INIT_QUAD_LDD_P
#else
#define STORE_ASI	0x80		/* ASI_P */
#endif
#endif

#ifndef STORE_MRU_ASI
#ifndef SIMULATE_NIAGARA_ON_NON_NIAGARA
#define STORE_MRU_ASI	ASI_ST_BLKINIT_MRU_P
#else
#define STORE_MRU_ASI	0x80		/* ASI_P */
#endif
#endif

#ifndef STORE_INIT
#define STORE_INIT(src,addr)	stxa src, [addr] STORE_ASI
#endif

#ifndef STORE_INIT_MRU
#define STORE_INIT_MRU(src,addr)	stxa src, [addr] STORE_MRU_ASI
#endif

#ifndef FUNC_NAME
#define FUNC_NAME	M7memcpy
#endif

#ifndef PREAMBLE
#define PREAMBLE
#endif

#define	BLOCK_SIZE	64
#define	SHORTCOPY	3
#define	SHORTCHECK	14
#define	SHORT_LONG	64	/* max copy for short longword-aligned case */
				/* must be at least 64 */
#define	SMALL_MAX	128
#define	MED_UMAX	1024	/* max copy for medium un-aligned case */
#define	MED_WMAX	1024	/* max copy for medium word-aligned case */
#define	MED_MAX		1024	/* max copy for medium longword-aligned case */
#define ST_CHUNK	24	/* ST_CHUNK - block of values for BIS Store */
#define ALIGN_PRE	24	/* distance for aligned prefetch loop */

	.register	%g2,#scratch

	.section	".text"
	.global		FUNC_NAME
	.type		FUNC_NAME, #function
	.align		16
FUNC_NAME:
	srlx            %o2, 31, %g2
	cmp             %g2, 0
	tne             %xcc, 5
	PREAMBLE
	mov		%o0, %g1	! save %o0
	brz,pn          %o2, .Lsmallx
	 cmp            %o2, 3
	ble,pn          %icc, .Ltiny_cp
	 cmp            %o2, 19
	ble,pn          %icc, .Lsmall_cp
	 or             %o0, %o1, %g2
	cmp             %o2, SMALL_MAX
	bl,pn           %icc, .Lmedium_cp
	 nop

.Lmedium:
	neg	%o0, %o5
	andcc	%o5, 7, %o5		! bytes till DST 8 byte aligned
	brz,pt	%o5, .Ldst_aligned_on_8

	! %o5 has the bytes to be written in partial store.
	 sub	%o2, %o5, %o2
	sub	%o1, %o0, %o1		! %o1 gets the difference
7:					! dst aligning loop
	add	%o1, %o0, %o4
	EX_LD(LOAD(ldub, %o4, %o4), memcpy_retl_o2_plus_o5)	! load one byte
	subcc	%o5, 1, %o5
	EX_ST(STORE(stb, %o4, %o0), memcpy_retl_o2_plus_o5_plus_1)
	bgu,pt	%xcc, 7b
	 add	%o0, 1, %o0		! advance dst
	add	%o1, %o0, %o1		! restore %o1
.Ldst_aligned_on_8:
	andcc	%o1, 7, %o5
	brnz,pt	%o5, .Lsrc_dst_unaligned_on_8
	 nop

.Lsrc_dst_aligned_on_8:
	! check if we are copying MED_MAX or more bytes
	set MED_MAX, %o3
	cmp %o2, %o3 			! limit to store buffer size
	bgu,pn	%xcc, .Llarge_align8_copy
	 nop

/*
 * Special case for handling when src and dest are both long word aligned
 * and total data to move is less than MED_MAX bytes
 */
.Lmedlong:
	subcc	%o2, 63, %o2		! adjust length to allow cc test
	ble,pn	%xcc, .Lmedl63		! skip big loop if less than 64 bytes
	 nop
.Lmedl64:
	EX_LD(LOAD(ldx, %o1, %o4), memcpy_retl_o2_plus_63)	! load
	subcc	%o2, 64, %o2		! decrement length count
	EX_ST(STORE(stx, %o4, %o0), memcpy_retl_o2_plus_63_64)	! and store
	EX_LD(LOAD(ldx, %o1+8, %o3), memcpy_retl_o2_plus_63_56)	! a block of 64
	EX_ST(STORE(stx, %o3, %o0+8), memcpy_retl_o2_plus_63_56)
	EX_LD(LOAD(ldx, %o1+16, %o4), memcpy_retl_o2_plus_63_48)
	EX_ST(STORE(stx, %o4,