fix(firebase_fdw): support nested collection

docs/firebase.md

@@ -101,6 +101,11 @@ The full list of foreign table options are below:
 
 - `object` - Object name in Firebase, required.
 
+  For Authenciation users, the object name is fixed to `auth/users`. For Firestore documents, its format is `firestore/<collection_id>`, note that collection id must be a full path id. For example,
+
+  - `firestore/my-collection`
+  - `firestore/my-collection/my-document/another-collection`
+
 ## Examples
 
 Some examples on how to use Firebase foreign tables.
@@ -118,7 +123,7 @@ create foreign table firebase_docs (
 )
   server firebase_server
   options (
-    object 'firestore/user-profiles'  -- format: 'firestore/[collection_id]'
+    object 'firestore/user-profiles'
   );
 ```
 

wrappers/.ci/docker-compose.yaml

@@ -37,18 +37,17 @@ services:
       retries: 20
 
   firebase:
-    image: andreysenov/firebase-tools:11.24.1-node-14-alpine
+    image: andreysenov/firebase-tools:12.9.1-node-18-slim
     container_name: firebase-wrapped
-    command: firebase emulators:start --project supa --only auth,firestore --import=/baseline-data
+    command: firebase emulators:start --project supa --only auth,firestore --import=/home/node/baseline-data
     volumes:
-      - ../dockerfiles/firebase/baseline-data:/baseline-data
-      - ../dockerfiles/firebase/firebase.json:/home/node/firebase.json
+      - ../dockerfiles/firebase:/home/node
     ports:
       - "4000:4000" # UI
       - "8080:8080" # Firestore
       - "9099:9099" # Auth
     healthcheck:
-      test: sleep 4 && wget --no-verbose --tries=1 --spider http://localhost:9099/ || exit 1
+      test: sleep 4 && (echo >/dev/tcp/127.0.0.1/9099) &>/dev/null || exit 1
       interval: 10s
       timeout: 5s
       retries: 30

wrappers/dockerfiles/firebase/README.md

@@ -0,0 +1,29 @@
+## Start Firebase Emulator
+
+ref: https://github.com/AndreySenov/firebase-tools-docker
+
+```bash
+cd wrappers
+
+docker run \
+  --name "firebase-emu" \
+  --rm \
+  -p=9000:9000 \
+  -p=8080:8080 \
+  -p=4000:4000 \
+  -p=9099:9099 \
+  -p=8085:8085 \
+  -p=5001:5001 \
+  -p=9199:9199 \
+  -v $PWD/dockerfiles/firebase:/home/node \
+  andreysenov/firebase-tools:12.9.1-node-18-slim \
+  firebase emulators:start --project supa --only auth,firestore --import=/home/node/baseline-data
+```
+
+## Export Firebase Data
+
+```bash
+docker exec -it "firebase-emu" bash
+
+firebase emulators:export ./baseline-data --project supa
+```

wrappers/dockerfiles/firebase/baseline-data/auth_export/accounts.json

@@ -1 +1 @@
-{"kind":"identitytoolkit#DownloadAccountResponse","users":[{"localId":"JeXJ2gUHCpYSQV6zXlRYfRYHUfMl","createdAt":"1668488830314","lastLoginAt":"1668488830310","displayName":"Bo Lu","photoUrl":"","emailVerified":false,"email":"[email protected]","salt":"fakeSaltNZhRYxXIjWaiXB75SD8z","passwordHash":"fakeHash:salt=fakeSaltNZhRYxXIjWaiXB75SD8z:password=adsf123","passwordUpdatedAt":1668488830313,"validSince":"1668488830","providerUserInfo":[{"providerId":"password","email":"[email protected]","federatedId":"[email protected]","rawId":"[email protected]","displayName":"Bo Lu","photoUrl":""}]},{"localId":"eUHS9EaQS2lZO0zbyMq28m02SUGP","createdAt":"1668488839113","lastLoginAt":"1668488839111","displayName":"Copple","photoUrl":"","emailVerified":false,"email":"[email protected]","salt":"fakeSaltL62cBYW58puy9DarO9TO","passwordHash":"fakeHash:salt=fakeSaltL62cBYW58puy9DarO9TO:password=asdf123","passwordUpdatedAt":1668488839113,"validSince":"1668488839","providerUserInfo":[{"providerId":"password","email":"[email protected]","federatedId":"[email protected]","rawId":"[email protected]","displayName":"Copple","photoUrl":""}]}]}
+{"kind":"identitytoolkit#DownloadAccountResponse","users":[{"localId":"JeXJ2gUHCpYSQV6zXlRYfRYHUfMl","createdAt":"1668488830314","lastLoginAt":"1668488830310","displayName":"Bo Lu","photoUrl":"","passwordHash":"fakeHash:salt=fakeSaltNZhRYxXIjWaiXB75SD8z:password=adsf123","salt":"fakeSaltNZhRYxXIjWaiXB75SD8z","passwordUpdatedAt":1701484518847,"providerUserInfo":[{"providerId":"password","email":"[email protected]","federatedId":"[email protected]","rawId":"[email protected]","displayName":"Bo Lu","photoUrl":""}],"validSince":"1701484518","email":"[email protected]","emailVerified":false,"disabled":false},{"localId":"eUHS9EaQS2lZO0zbyMq28m02SUGP","createdAt":"1668488839113","lastLoginAt":"1668488839111","displayName":"Copple","photoUrl":"","passwordHash":"fakeHash:salt=fakeSaltL62cBYW58puy9DarO9TO:password=asdf123","salt":"fakeSaltL62cBYW58puy9DarO9TO","passwordUpdatedAt":1701484518848,"providerUserInfo":[{"providerId":"password","email":"[email protected]","federatedId":"[email protected]","rawId":"[email protected]","displayName":"Copple","photoUrl":""}],"validSince":"1701484518","email":"[email protected]","emailVerified":false,"disabled":false}]}

wrappers/dockerfiles/firebase/baseline-data/firebase-export-metadata.json

@@ -1,20 +1,12 @@
 {
-  "version": "9.17.0",
+  "version": "11.16.0",
   "firestore": {
-    "version": "1.13.1",
+    "version": "1.15.1",
     "path": "firestore_export",
     "metadata_file": "firestore_export/firestore_export.overall_export_metadata"
   },
-  "database": {
-    "version": "4.7.2",
-    "path": "database_export"
-  },
   "auth": {
-    "version": "9.17.0",
+    "version": "11.16.0",
     "path": "auth_export"
-  },
-  "storage": {
-    "version": "9.17.0",
-    "path": "storage_export"
   }
 }

wrappers/src/fdw/firebase_fdw/firebase_fdw.rs

@@ -183,7 +183,7 @@ impl FirebaseFdw {
             _ => {
                 // match for firestore documents
                 // ref: https://firebase.google.com/docs/firestore/reference/rest/v1beta1/projects.databases.documents/listDocuments
-                let re = Regex::new(r"^firestore/(?P<collection>[^/]+)").expect("regex is valid");
+                let re = Regex::new(r"^firestore/(?P<collection>.+)").expect("regex is valid");
                 if let Some(caps) = re.captures(obj) {
                     let base_url =
                         require_option_or("base_url", options, Self::DEFAULT_FIRESTORE_BASE_URL);

wrappers/src/fdw/firebase_fdw/tests.rs

@@ -95,6 +95,41 @@ mod tests {
                 vec![
                 ("projects/supa/databases/(default)/documents/my-collection/bSMScXpZHMJe9ilE9Yqs",
                  serde_json::json!({"id": {"integerValue": "1"}, "name": {"stringValue": "hello"}}))]);
+
+            c.update(
+                r#"
+                CREATE FOREIGN TABLE firebase_docs_nested (
+                  name text,
+                  fields jsonb,
+                  create_time timestamp,
+                  update_time timestamp
+                )
+                SERVER my_firebase_server
+                OPTIONS (
+                  object 'firestore/my-collection/bSMScXpZHMJe9ilE9Yqs/my-collection2',
+                  base_url 'http://localhost:8080/v1/projects'
+                )
+             "#,
+                None,
+                None,
+            )
+            .unwrap();
+
+            let results = c
+                .select("SELECT name,fields FROM firebase_docs_nested", None, None)
+                .unwrap()
+                .filter_map(|r| {
+                    r.get_by_name::<&str, _>("name")
+                        .unwrap()
+                        .zip(r.get_by_name::<JsonB, _>("fields").unwrap().map(|j| j.0))
+                })
+                .collect::<Vec<_>>();
+
+            assert_eq!(
+                results,
+                vec![
+                ("projects/supa/databases/(default)/documents/my-collection/bSMScXpZHMJe9ilE9Yqs/my-collection2/fkSWL4hNJ3lRc1ZIorPm",
+                 serde_json::json!({"foo": {"stringValue": "bar"}}))]);
         });
     }
 }