s390/lib: use expoline for all bcr instructions

The memove, memset, memcpy, __memset16, __memset32 and __memset64
function have an additional indirect return branch in form of a
"bzr" instruction. These need to use expolines as well.

Cc: <stable@vger.kernel.org> # v4.17+
Fixes: 97489e0663 ("s390/lib: use expoline for indirect branches")
Reviewed-by: Heiko Carstens <heiko.carstens@de.ibm.com>
Signed-off-by: Martin Schwidefsky <schwidefsky@de.ibm.com>

diff --git a/arch/s390/lib/mem.S b/arch/s390/lib/mem.S
index 2311f15be9cf04b3bad1f766453e6ff970d53a83..40c4d59c926e52d8a7f3e7c3870dbb69e8091c8b 100644 (file)
--- a/arch/s390/lib/mem.S
+++ b/arch/s390/lib/mem.S
@@ -17,7 +17,7 @@
 ENTRY(memmove)
        ltgr    %r4,%r4
        lgr     %r1,%r2
-       bzr     %r14
+       jz      .Lmemmove_exit
        aghi    %r4,-1
        clgr    %r2,%r3
        jnh     .Lmemmove_forward
@@ -36,6 +36,7 @@ ENTRY(memmove)
 .Lmemmove_forward_remainder:
        larl    %r5,.Lmemmove_mvc
        ex      %r4,0(%r5)
+.Lmemmove_exit:
        BR_EX   %r14
 .Lmemmove_reverse:
        ic      %r0,0(%r4,%r3)
@@ -65,7 +66,7 @@ EXPORT_SYMBOL(memmove)
  */
 ENTRY(memset)
        ltgr    %r4,%r4
-       bzr     %r14
+       jz      .Lmemset_exit
        ltgr    %r3,%r3
        jnz     .Lmemset_fill
        aghi    %r4,-1
@@ -80,6 +81,7 @@ ENTRY(memset)
 .Lmemset_clear_remainder:
        larl    %r3,.Lmemset_xc
        ex      %r4,0(%r3)
+.Lmemset_exit:
        BR_EX   %r14
 .Lmemset_fill:
        cghi    %r4,1
@@ -115,7 +117,7 @@ EXPORT_SYMBOL(memset)
  */
 ENTRY(memcpy)
        ltgr    %r4,%r4
-       bzr     %r14
+       jz      .Lmemcpy_exit
        aghi    %r4,-1
        srlg    %r5,%r4,8
        ltgr    %r5,%r5
@@ -124,6 +126,7 @@ ENTRY(memcpy)
 .Lmemcpy_remainder:
        larl    %r5,.Lmemcpy_mvc
        ex      %r4,0(%r5)
+.Lmemcpy_exit:
        BR_EX   %r14
 .Lmemcpy_loop:
        mvc     0(256,%r1),0(%r3)
@@ -145,9 +148,9 @@ EXPORT_SYMBOL(memcpy)
 .macro __MEMSET bits,bytes,insn
 ENTRY(__memset\bits)
        ltgr    %r4,%r4
-       bzr     %r14
+       jz      .L__memset_exit\bits
        cghi    %r4,\bytes
-       je      .L__memset_exit\bits
+       je      .L__memset_store\bits
        aghi    %r4,-(\bytes+1)
        srlg    %r5,%r4,8
        ltgr    %r5,%r5
@@ -163,8 +166,9 @@ ENTRY(__memset\bits)
        larl    %r5,.L__memset_mvc\bits
        ex      %r4,0(%r5)
        BR_EX   %r14
-.L__memset_exit\bits:
+.L__memset_store\bits:
        \insn   %r3,0(%r2)
+.L__memset_exit\bits:
        BR_EX   %r14
 .L__memset_mvc\bits:
        mvc     \bytes(1,%r1),0(%r1)