drm/i915: Align start for memcpy_from_wc

The movntqda requires 16-byte alignment for the source pointer. Avoid
falling back to clflush if the source pointer is misaligned by doing the
doing a small uncached memcpy to fixup the alignments.

v2: Turn the unaligned copy into a genuine helper

Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Reviewed-by: Joonas Lahtinen <joonas.lahtinen@linux.intel.com>
Link: https://patchwork.freedesktop.org/patch/msgid/20191211110437.4082687-5-chris@chris-wilson.co.uk

diff --git a/drivers/gpu/drm/i915/i915_cmd_parser.c b/drivers/gpu/drm/i915/i915_cmd_parser.c
index b692c40348a161a58171e7a6157434200b63d1ea..ddc8bf5175d98ee6d92011f31455de4430ea8c18 100644 (file)
--- a/drivers/gpu/drm/i915/i915_cmd_parser.c
+++ b/drivers/gpu/drm/i915/i915_cmd_parser.c
@@ -1151,13 +1151,12 @@ static u32 *copy_batch(struct drm_i915_gem_object *dst_obj,
        }
 
        src = ERR_PTR(-ENODEV);
-       if (src_needs_clflush &&
-           i915_can_memcpy_from_wc(NULL, offset, 0)) {
+       if (src_needs_clflush && i915_has_memcpy_from_wc()) {
                src = i915_gem_object_pin_map(src_obj, I915_MAP_WC);
                if (!IS_ERR(src)) {
-                       i915_memcpy_from_wc(dst,
-                                           src + offset,
-                                           ALIGN(length, 16));
+                       i915_unaligned_memcpy_from_wc(dst,
+                                                     src + offset,
+                                                     length);
                        i915_gem_object_unpin_map(src_obj);
                }
        }

diff --git a/drivers/gpu/drm/i915/i915_memcpy.c b/drivers/gpu/drm/i915/i915_memcpy.c
index 07b04b0acb773029b3b0a23711610b89825b4141..fdd550405fd3d2cc052c3d31e43c597f0bba0073 100644 (file)
--- a/drivers/gpu/drm/i915/i915_memcpy.c
+++ b/drivers/gpu/drm/i915/i915_memcpy.c
@@ -27,6 +27,12 @@
 
 #include "i915_memcpy.h"
 
+#if IS_ENABLED(CONFIG_DRM_I915_DEBUG)
+#define CI_BUG_ON(expr) BUG_ON(expr)
+#else
+#define CI_BUG_ON(expr) BUILD_BUG_ON_INVALID(expr)
+#endif
+
 static DEFINE_STATIC_KEY_FALSE(has_movntdqa);
 
 #ifdef CONFIG_AS_MOVNTDQA
@@ -34,7 +40,6 @@ static void __memcpy_ntdqa(void *dst, const void *src, unsigned long len)
 {
        kernel_fpu_begin();
 
-       len >>= 4;
        while (len >= 4) {
                asm("movntdqa   (%0), %%xmm0\n"
                    "movntdqa 16(%0), %%xmm1\n"
@@ -59,6 +64,38 @@ static void __memcpy_ntdqa(void *dst, const void *src, unsigned long len)
 
        kernel_fpu_end();
 }
+
+static void __memcpy_ntdqu(void *dst, const void *src, unsigned long len)
+{
+       kernel_fpu_begin();
+
+       while (len >= 4) {
+               asm("movntdqa   (%0), %%xmm0\n"
+                   "movntdqa 16(%0), %%xmm1\n"
+                   "movntdqa 32(%0), %%xmm2\n"
+                   "movntdqa 48(%0), %%xmm3\n"
+                   "movups %%xmm0,   (%1)\n"
+                   "movups %%xmm1, 16(%1)\n"
+                   "movups %%xmm2, 32(%1)\n"
+                   "movups %%xmm3, 48(%1)\n"
+                   :: "r" (src), "r" (dst) : "memory");
+               src += 64;
+               dst += 64;
+               len -= 4;
+       }
+       while (len--) {
+               asm("movntdqa (%0), %%xmm0\n"
+                   "movups %%xmm0, (%1)\n"
+                   :: "r" (src), "r" (dst) : "memory");
+               src += 16;
+               dst += 16;
+       }
+
+       kernel_fpu_end();
+}
+#else
+static void __memcpy_ntdqa(void *dst, const void *src, unsigned long len) {}
+static void __memcpy_ntdqu(void *dst, const void *src, unsigned long len) {}
 #endif
 
 /**
@@ -83,17 +120,47 @@ bool i915_memcpy_from_wc(void *dst, const void *src, unsigned long len)
        if (unlikely(((unsigned long)dst | (unsigned long)src | len) & 15))
                return false;
 
-#ifdef CONFIG_AS_MOVNTDQA
        if (static_branch_likely(&has_movntdqa)) {
                if (likely(len))
-                       __memcpy_ntdqa(dst, src, len);
+                       __memcpy_ntdqa(dst, src, len >> 4);
                return true;
        }
-#endif
 
        return false;
 }
 
+/**
+ * i915_unaligned_memcpy_from_wc: perform a mostly accelerated read from WC
+ * @dst: destination pointer
+ * @src: source pointer
+ * @len: how many bytes to copy
+ *
+ * Like i915_memcpy_from_wc(), the unaligned variant copies @len bytes from
+ * @src to @dst using * non-temporal instructions where available, but
+ * accepts that its arguments may not be aligned, but are valid for the
+ * potential 16-byte read past the end.
+ */
+void i915_unaligned_memcpy_from_wc(void *dst, void *src, unsigned long len)
+{
+       unsigned long addr;
+
+       CI_BUG_ON(!i915_has_memcpy_from_wc());
+
+       addr = (unsigned long)src;
+       if (!IS_ALIGNED(addr, 16)) {
+               unsigned long x = min(ALIGN(addr, 16) - addr, len);
+
+               memcpy(dst, src, x);
+
+               len -= x;
+               dst += x;
+               src += x;
+       }
+
+       if (likely(len))
+               __memcpy_ntdqu(dst, src, DIV_ROUND_UP(len, 16));
+}
+
 void i915_memcpy_init_early(struct drm_i915_private *dev_priv)
 {
        /*

diff --git a/drivers/gpu/drm/i915/i915_memcpy.h b/drivers/gpu/drm/i915/i915_memcpy.h
index 970d84b1698766abe280ec2c4ee52b0c3f55b05d..e36d30edd98774034e4bdc9171111e820d7c95c7 100644 (file)
--- a/drivers/gpu/drm/i915/i915_memcpy.h
+++ b/drivers/gpu/drm/i915/i915_memcpy.h
@@ -11,7 +11,9 @@
 struct drm_i915_private;
 
 void i915_memcpy_init_early(struct drm_i915_private *i915);
+
 bool i915_memcpy_from_wc(void *dst, const void *src, unsigned long len);
+void i915_unaligned_memcpy_from_wc(void *dst, void *src, unsigned long len);
 
 /* The movntdqa instructions used for memcpy-from-wc require 16-byte alignment,
  * as well as SSE4.1 support. i915_memcpy_from_wc() will report if it cannot