ci: run the gated redis store tests (#10746)

* redis2: route the orphan cleanup existence checks to the master

* scaffold: the redis_cluster2 read routing key is useReadOnly

* ci: run the gated redis store tests

* redis2: poll for the redis expiry instead of a fixed sleep

* redis2: assert the value key exists before testing its expiry

.github/workflows/go.yml

@@ -111,6 +111,16 @@ jobs:
   test:
     name: Test
     runs-on: ubuntu-latest
+    services:
+      redis:
+        image: redis:8
+        ports:
+          - 6379:6379
+        options: >-
+          --health-cmd "redis-cli ping"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
     steps:
     - name: Check out code into the Go module directory
       uses: actions/checkout@v7
@@ -119,6 +129,8 @@ jobs:
       with:
         go-version-file: 'go.mod'
     - name: Test
+      env:
+        RUN_REDIS_TESTS: "1"
       run: cd weed; go test -tags "elastic gocdk sqlite ydb tarantool tikv rclone" -v ./...
 
   test-32bit:

weed/filer/redis2/universal_redis_store_test.go

@@ -181,17 +181,33 @@ func TestRemoveOrphanedDirectoryListMemberSkipsSuperLargeDirectory(t *testing.T)
 	}
 }
 
+func requireValueKey(t *testing.T, store *UniversalRedis2Store, path util.FullPath) {
+	t.Helper()
+
+	if exists, err := store.Client.Exists(context.Background(), store.getKey(string(path))).Result(); err != nil || exists != 1 {
+		t.Fatalf("value key %s exists=%d err=%v, want it present", path, exists, err)
+	}
+}
+
 func TestListDirectoryEntriesRemovesIndexMembersExpiredByRedis(t *testing.T) {
 	store, dir := newTestStore(t, "")
 
 	insertTestEntry(t, store, dir.Child("ttl"), 1)
+	requireValueKey(t, store, dir.Child("ttl"))
 
-	time.Sleep(1500 * time.Millisecond)
+	deadline := time.Now().Add(5 * time.Second)
-
+	for {
-	if exists, err := store.Client.Exists(context.Background(), store.getKey(string(dir.Child("ttl")))).Result(); err != nil {
+		exists, err := store.Client.Exists(context.Background(), store.getKey(string(dir.Child("ttl")))).Result()
-		t.Fatalf("check value key: %v", err)
+		if err != nil {
-	} else if exists != 0 {
+			t.Fatalf("check value key: %v", err)
-		t.Fatal("redis did not expire the value key, the logical expiry path is not being bypassed")
+		}
+		if exists == 0 {
+			break
+		}
+		if time.Now().After(deadline) {
+			t.Fatal("redis did not expire the value key, the logical expiry path is not being bypassed")
+		}
+		time.Sleep(50 * time.Millisecond)
 	}
 
 	if names := listNames(t, store, dir); len(names) != 0 {
@@ -223,6 +239,7 @@ func TestListDirectoryEntriesDeletesLogicallyExpiredEntries(t *testing.T) {
 			store, dir := newTestStore(t, keyPrefix)
 
 			insertLogicallyExpiredTestEntry(t, store, dir.Child("stale"))
+			requireValueKey(t, store, dir.Child("stale"))
 
 			if names := listNames(t, store, dir); len(names) != 0 {
 				t.Fatalf("listed %v, want none", names)