gru: fix prefetch and speculation bugs

Fix several bugs related to prefetch, ordering & speculation:

	- GRU cch_allocate() instruction causes cacheable memory
	  to be created. Add a barriers to prevent speculation
	  from prefetching data before it exists.
	- Add memory barriers before cache-flush instructions to ensure
	  that previously stored data is included in the line flushed to memory.

Signed-off-by: Jack Steiner <steiner@sgi.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

diff --git a/drivers/misc/sgi-gru/gru_instructions.h b/drivers/misc/sgi-gru/gru_instructions.h
index e033b6ce4a3f637e1f928507841269e2756bfa48..32f358d9f4290312dd420f50753238ba2d5e4bc5 100644 (file)
--- a/drivers/misc/sgi-gru/gru_instructions.h
+++ b/drivers/misc/sgi-gru/gru_instructions.h
@@ -325,6 +325,7 @@ static inline void gru_flush_cache(void *p)
 static inline void gru_start_instruction(struct gru_instruction *ins, int op32)
 {
        gru_ordered_store_int(ins, op32);
+       mb();
        gru_flush_cache(ins);
 }
 

diff --git a/drivers/misc/sgi-gru/grufault.c b/drivers/misc/sgi-gru/grufault.c
index 7466234450f347eb8d8caf705bb4a26837e10ae4..d3cacd696b38df6764c8cc1e961b5cf3eb9626d1 100644 (file)
--- a/drivers/misc/sgi-gru/grufault.c
+++ b/drivers/misc/sgi-gru/grufault.c
@@ -333,6 +333,7 @@ static int gru_try_dropin(struct gru_thread_state *gts,
         */
        if (tfh->status != TFHSTATUS_EXCEPTION) {
                gru_flush_cache(tfh);
+               sync_core();
                if (tfh->status != TFHSTATUS_EXCEPTION)
                        goto failnoexception;
                STAT(tfh_stale_on_fault);
@@ -599,6 +600,7 @@ int gru_get_exception_detail(unsigned long arg)
                cbrnum = thread_cbr_number(gts, ucbnum);
                cbe = get_cbe_by_index(gts->ts_gru, cbrnum);
                gru_flush_cache(cbe);   /* CBE not coherent */
+               sync_core();            /* make sure we are have current data */
                excdet.opc = cbe->opccpy;
                excdet.exopc = cbe->exopccpy;
                excdet.ecause = cbe->ecause;

diff --git a/drivers/misc/sgi-gru/gruhandles.c b/drivers/misc/sgi-gru/gruhandles.c
index 806419a6b44c3adc405731a8310c4b366d2f7990..f1117a7637a21465e338a87f4971946ca836e6cc 100644 (file)
--- a/drivers/misc/sgi-gru/gruhandles.c
+++ b/drivers/misc/sgi-gru/gruhandles.c
@@ -91,9 +91,18 @@ static int wait_instruction_complete(void *h, enum mcs_op opc)
 
 int cch_allocate(struct gru_context_configuration_handle *cch)
 {
+       int ret;
+
        cch->opc = CCHOP_ALLOCATE;
        start_instruction(cch);
-       return wait_instruction_complete(cch, cchop_allocate);
+       ret = wait_instruction_complete(cch, cchop_allocate);
+
+       /*
+        * Stop speculation into the GSEG being mapped by the previous ALLOCATE.
+        * The GSEG memory does not exist until the ALLOCATE completes.
+        */
+       sync_core();
+       return ret;
 }
 
 int cch_start(struct gru_context_configuration_handle *cch)
@@ -112,9 +121,18 @@ int cch_interrupt(struct gru_context_configuration_handle *cch)
 
 int cch_deallocate(struct gru_context_configuration_handle *cch)
 {
+       int ret;
+
        cch->opc = CCHOP_DEALLOCATE;
        start_instruction(cch);
-       return wait_instruction_complete(cch, cchop_deallocate);
+       ret = wait_instruction_complete(cch, cchop_deallocate);
+
+       /*
+        * Stop speculation into the GSEG being unmapped by the previous
+        * DEALLOCATE.
+        */
+       sync_core();
+       return ret;
 }
 
 int cch_interrupt_sync(struct gru_context_configuration_handle

diff --git a/drivers/misc/sgi-gru/grukservices.c b/drivers/misc/sgi-gru/grukservices.c
index 24ec109e61cc640783b7051e15248995a2af807e..8c81aca0463ae1a21d6d8fd4badd461370a83997 100644 (file)
--- a/drivers/misc/sgi-gru/grukservices.c
+++ b/drivers/misc/sgi-gru/grukservices.c
@@ -395,6 +395,7 @@ int gru_get_cb_exception_detail(void *cb,
        cbrnum = thread_cbr_number(bs->bs_kgts, get_cb_number(cb));
        cbe = get_cbe(GRUBASE(cb), cbrnum);
        gru_flush_cache(cbe);   /* CBE not coherent */
+       sync_core();
        excdet->opc = cbe->opccpy;
        excdet->exopc = cbe->exopccpy;
        excdet->ecause = cbe->ecause;
@@ -461,9 +462,10 @@ int gru_check_status_proc(void *cb)
        int ret;
 
        ret = gen->istatus;
-       if (ret != CBS_EXCEPTION)
-               return ret;
-       return gru_retry_exception(cb);
+       if (ret == CBS_EXCEPTION)
+               ret = gru_retry_exception(cb);
+       rmb();
+       return ret;
 
 }
 
@@ -475,7 +477,7 @@ int gru_wait_proc(void *cb)
        ret = gru_wait_idle_or_exception(gen);
        if (ret == CBS_EXCEPTION)
                ret = gru_retry_exception(cb);
-
+       rmb();
        return ret;
 }
 

diff --git a/drivers/misc/sgi-gru/grumain.c b/drivers/misc/sgi-gru/grumain.c
index 944028871884df122ab5a7a932786c733150c596..a383271d391240a4042dfee31869b6b113437a3e 100644 (file)
--- a/drivers/misc/sgi-gru/grumain.c
+++ b/drivers/misc/sgi-gru/grumain.c
@@ -499,6 +499,9 @@ static void gru_load_context_data(void *save, void *grubase, int ctxnum,
                        memset(cbe + i * GRU_HANDLE_STRIDE, 0,
                                                GRU_CACHE_LINE_BYTES);
                }
+               /* Flush CBE to hide race in context restart */
+               mb();
+               gru_flush_cache(cbe + i * GRU_HANDLE_STRIDE);
                cb += GRU_HANDLE_STRIDE;
        }
 
@@ -519,6 +522,12 @@ static void gru_unload_context_data(void *save, void *grubase, int ctxnum,
        cb = gseg + GRU_CB_BASE;
        cbe = grubase + GRU_CBE_BASE;
        length = hweight64(dsrmap) * GRU_DSR_AU_BYTES;
+
+       /* CBEs may not be coherent. Flush them from cache */
+       for_each_cbr_in_allocation_map(i, &cbrmap, scr)
+               gru_flush_cache(cbe + i * GRU_HANDLE_STRIDE);
+       mb();           /* Let the CL flush complete */
+
        gru_prefetch_context(gseg, cb, cbe, cbrmap, length);
 
        for_each_cbr_in_allocation_map(i, &cbrmap, scr) {