/*
* Copyright © 2017 Broadcom
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice (including the next
* paragraph) shall be included in all copies or substantial portions of the
* Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
* IN THE SOFTWARE.
*/
/** @file v3d_cpu_tiling.h
*
* Contains load/store functions common to both v3d and vc4. The utile layout
* stayed the same, though the way utiles get laid out has changed.
*/
static inline void
v3d_load_utile(void *cpu, uint32_t cpu_stride,
void *gpu, uint32_t gpu_stride)
{
#if defined(V3D_BUILD_NEON) && defined(PIPE_ARCH_ARM)
if (gpu_stride == 8) {
__asm__ volatile (
/* Load from the GPU in one shot, no interleave, to
* d0-d7.
*/
"vldm %[gpu], {q0, q1, q2, q3}\n"
/* Store each 8-byte line to cpu-side destination,
* incrementing it by the stride each time.
*/
"vst1.8 d0, [%[cpu]], %[cpu_stride]\n"
"vst1.8 d1, [%[cpu]], %[cpu_stride]\n"
"vst1.8 d2, [%[cpu]], %[cpu_stride]\n"
"vst1.8 d3, [%[cpu]], %[cpu_stride]\n"
"vst1.8 d4, [%[cpu]], %[cpu_stride]\n"
"vst1.8 d5, [%[cpu]], %[cpu_stride]\n"
"vst1.8 d6, [%[cpu]], %[cpu_stride]\n"
"vst1.8 d7, [%[cpu]]\n"
: [cpu] "+r"(cpu)
: [gpu] "r"(gpu),
[cpu_stride] "r"(cpu_stride)
: "q0", "q1", "q2", "q3");
return;
} else if (gpu_stride == 16) {
void *cpu2 = cpu + 8;
__asm__ volatile (
/* Load from the GPU in one shot, no interleave, to
* d0-d7.
*/
"vldm %[gpu], {q0, q1, q2, q3};\n"
/* Store each 16-byte line in 2 parts to the cpu-side
* destination. (vld1 can only store one d-register
* at a time).
*/
"vst1.8 d0, [%[cpu]], %[cpu_stride]\n"
"vst1.8 d1, [%[cpu2]],%[cpu_stride]\n"
"vst1.8 d2, [%[cpu]], %[cpu_stride]\n"
"vst1.8 d3, [%[cpu2]],%[cpu_stride]\n"
"vst1.8 d4, [%[cpu]], %[cpu_stride]\n"
"vst1.8 d5, [%[cpu2]],%[cpu_stride]\n"
"vst1.8 d6, [%[cpu]]\n"
"vst1.8 d7, [%[cpu2]]\n"
: [cpu] "+r"(cpu),
[cpu2] "+r"(cpu2)
: [gpu] "r"(gpu),
[cpu_stride] "r"(cpu_stride)
: "q0", "q1", "q2", "q3");
return;
}
#elif defined (PIPE_ARCH_AARCH64)
if (gpu_stride == 8) {
__asm__ volatile (
/* Load from the GPU in one shot, no interleave, to
* d0-d7.
*/
"ld1 {v0.2d, v1.2d, v2.2d, v3.2d}, [%[gpu]]\n"
/* Store each 8-byte line to cpu-side destination,
* incrementing it by the stride each time.
*/
"st1 {v0.D}[0], [%[cpu]], %[cpu_stride]\n"
"st1 {v0.D}[1], [%[cpu]], %[cpu_stride]\n"
"st1 {v1.D}[0], [%[cpu]], %[cpu_stride]\n"
"st1 {v1.D}[1], [%[cpu]], %[cpu_stride]\n"
"st1 {v2.D}[0], [%[cpu]], %[cpu_stride]\n"
"st1 {v2.D}[1], [%[cpu]], %[cpu_stride]\n"
"st1 {v3.D}[0], [%[cpu]], %[cpu_stride]\n"
"st1 {v3.D}[1], [%[cpu]]\n"
: [cpu] "+r"(cpu)
: [gpu] "r"(gpu),
[cpu_stride] "r"(cpu_stride)
: "v0", "v1", "v2", "v3");
return;
} else if (gpu_stride == 16) {
void *cpu2 = cpu + 8;
__asm__ volatile (
/* Load from the GPU in one shot, no interleave, to
* d0-d7.
*/
"ld1 {v0.2d, v1.2d, v2.2d, v3.2d}, [%[gpu]]\n"
/* Store each 16-byte line in 2 parts to the cpu-side
* destination. (vld1 can only store one d-register
* at a time).
*/
"st1 {v0.D}[0], [%[cpu]], %[cpu_stride]\n"
"st1 {v0.D}[1], [%[cpu2]],%[cpu_stride]\n"
"st1 {v1.D}[0], [%[cpu]], %[cpu_stride]\n"
"st1 {v1.D}[1], [%[cpu2]],%[cpu_stride]\n"
"st1 {v2.D}[0], [%[cpu]], %[cpu_stride]\n"
"st1 {v2.D}[1], [%[cpu2]],%[cpu_stride]\n"
"st1 {v3.D}[0], [%[cpu]]\n"
"st1 {v3.D}[1], [%[cpu2]]\n"
: [cpu] "+r"(cpu),
[cpu2] "+r"(cpu2)
: [gpu] "r"(gpu),
[cpu_stride] "r"(cpu_stride)
: "v0", "v1", "v2", "v3");
return;
}
#endif
for (uint32_t gpu_offset = 0; gpu_offset < 64; gpu_offset += gpu_stride) {
memcpy(cpu, gpu + gpu_offset, gpu_stride);
cpu += cpu_stride;
}
}
static inline void
v3d_store_utile(void *gpu, uint32_t gpu_stride,
void *cpu, uint32_t cpu_stride)
{
#if defined(V3D_BUILD_NEON) && defined(PIPE_ARCH_ARM)
if (gpu_stride == 8) {
__asm__ volatile (
/* Load each 8-byte line from cpu-side source,
* incrementing it by the stride each time.
*/
"vld1.8 d0, [%[cpu]], %[cpu_stride]\n"
"vld1.8 d1, [%[cpu]], %[cpu_stride]\n"
"vld1.8 d2, [%[cpu]], %[cpu_stride]\n"
"vld1.8 d3, [%[cpu]], %[cpu_stride]\n"
"vld1.8 d4, [%[cpu]], %[cpu_stride]\n"
"vld1.8 d5, [%[cpu]], %[cpu_stride]\n"
"vld1.8 d6, [%[cpu]], %[cpu_stride]\n"
"vld1.8 d7, [%[cpu]]\n"
/* Load from the GPU in one shot, no interleave, to
* d0-d7.
*/
"vstm %[gpu], {q0, q1, q2, q3}\n"
: [cpu] "+r"(cpu)
: [gpu] "r"(gpu),
[cpu_stride] "r"(cpu_stride)
: "q0", "q1", "q2", "q3");
return;
} else if (gpu_stride == 16) {
void *cpu2 = cpu + 8;
__asm__ volatile (
/* Load each 16-byte line in 2 parts from the cpu-side
* destination. (vld1 can only store one d-register
* at a time).
*/
"vld1.8 d0, [%[cpu]], %[cpu_stride]\n"
"vld1.8 d1, [%[cpu2]],%[cpu_stride]\n"
"vld1.8 d2, [%[cpu]], %[cpu_stride]\n"
"vld1.8 d3, [%[cpu2]],%[cpu_stride]\n"
"vld1.8 d4, [%[cpu]], %[cpu_stride]\n"
"vld1.8 d5, [%[cpu2]],%[cpu_stride]\n"
"vld1.8 d6, [%[cpu]]\n"
"vld1.8 d7, [%[cpu2]]\n"
/* Store to the GPU in one shot, no interleave. */
"vstm %[gpu], {q0, q1, q2, q3}\n"
: [cpu] "+r"(cpu),
[cpu2] "+r"(cpu2)
: [gpu] "r"(gpu),
[cpu_stride] "r"(cpu_stride)
: "q0", "q1", "q2", "q3");
return;
}
#elif defined (PIPE_ARCH_AARCH64)
if (gpu_stride == 8) {
__asm__ volatile (
/* Load each 8-byte line from cpu-side source,
* incrementing it by the stride each time.
*/
"ld1 {v0.D}[0], [%[cpu]], %[cpu_stride]\n"
"ld1 {v0.D}[1], [%[cpu]], %[cpu_stride]\n"
"ld1 {v1.D}[0], [%[cpu]], %[cpu_stride]\n"
"ld1 {v1.D}[1], [%[cpu]], %[cpu_stride]\n"
"ld1 {v2.D}[0], [%[cpu]], %[cpu_stride]\n"
"ld1 {v2.D}[1], [%[cpu]], %[cpu_stride]\n"
"ld1 {v3.D}[0], [%[cpu]], %[cpu_stride]\n"
"ld1 {v3.D}[1], [%[cpu]]\n"
/* Store to the GPU in one shot, no interleave. */
"st1 {v0.2d, v1.2d, v2.2d, v3.2d}, [%[gpu]]\n"
: [cpu] "+r"(cpu)
: [gpu] "r"(gpu),
[cpu_stride] "r"(cpu_stride)
: "v0", "v1", "v2", "v3");
return;
} else if (gpu_stride == 16) {
void *cpu2 = cpu + 8;
__asm__ volatile (
/* Load each 16-byte line in 2 parts from the cpu-side
* destination. (vld1 can only store one d-register
* at a time).
*/
"ld1 {v0.D}[0], [%[cpu]], %[cpu_stride]\n"
"ld1 {v0.D}[1], [%[cpu2]],%[cpu_stride]\n"
"ld1 {v1.D}[0], [%[cpu]], %[cpu_stride]\n"
"ld1 {v1.D}[1], [%[cpu2]],%[cpu_stride]\n"
"ld1 {v2.D}[0], [%[cpu]], %[cpu_stride]\n"
"ld1 {v2.D}[1], [%[cpu2]],%[cpu_stride]\n"
"ld1 {v3.D}[0], [%[cpu]]\n"
"ld1 {v3.D}[1], [%[cpu2]]\n"
/* Store to the GPU in one shot, no interleave. */
"st1 {v0.2d, v1.2d, v2.2d, v3.2d}, [%[gpu]]\n"
: [cpu] "+r"(cpu),
[cpu2] "+r"(cpu2)
: [gpu] "r"(gpu),
[cpu_stride] "r"(cpu_stride)
: "v0", "v1", "v2", "v3");
return;
}
#endif
for (uint32_t gpu_offset = 0; gpu_offset < 64; gpu_offset += gpu_stride) {
memcpy(gpu + gpu_offset, cpu, gpu_stride);
cpu += cpu_stride;
}
}