Merge pull request #650 from akto-api-security/feature/ab_6

Feature/ab 6
akto-api-security · Mar 7, 2023 · 6eccc1c · 6eccc1c
2 parents 58b6ac3 + bc4cc49
commit 6eccc1c
Show file tree

Hide file tree

Showing 15 changed files with 379 additions and 36 deletions.
diff --git a/apps/dashboard/src/main/java/com/akto/action/ExportSampleDataAction.java b/apps/dashboard/src/main/java/com/akto/action/ExportSampleDataAction.java
@@ -45,7 +45,7 @@ public String importInBurp() {
 
         int apiCollectionId = apiCollection.getId();
 
-        List<SampleData> sampleDataList = SampleDataDao.instance.fetchSampleDataPaginated(apiCollectionId, lastUrlFetched, lastMethodFetched, limit);
+        List<SampleData> sampleDataList = SampleDataDao.instance.fetchSampleDataPaginated(apiCollectionId, lastUrlFetched, lastMethodFetched, limit, 1);
 
         lastMethodFetched = null;
         lastUrlFetched = null;

diff --git a/apps/dashboard/src/main/java/com/akto/action/OpenApiAction.java b/apps/dashboard/src/main/java/com/akto/action/OpenApiAction.java
@@ -27,20 +27,33 @@ public class OpenApiAction extends UserAction implements ServletResponseAware {
     private int apiCollectionId;
     private String openAPIString = null;
     private boolean includeHeaders = true;
+
+    private String lastFetchedUrl;
+    private String lastFetchedMethod;
     @Override
     public String execute() {
         try {
-
-            List<SampleData> sampleData = SampleDataDao.instance.findAll(
-                Filters.eq("_id.apiCollectionId", apiCollectionId)
-            );
             ApiCollection apiCollection = ApiCollectionsDao.instance.findOne("_id", apiCollectionId);
-            if (apiCollection == null) {
-                return ERROR.toUpperCase();
-            }
+            if (apiCollection == null) return ERROR.toUpperCase();
             String host =  apiCollection.getHostName();
+
+            int limit = 200;
+            List<SampleData> sampleDataList = SampleDataDao.instance.fetchSampleDataPaginated(
+                    apiCollectionId, lastFetchedUrl, lastFetchedMethod, limit, 1
+            );
+
+            int size = sampleDataList.size();
+            if (size < limit) {
+                lastFetchedUrl = null;
+                lastFetchedMethod = null;
+            } else {
+                SampleData last = sampleDataList.get(size-1);
+                lastFetchedUrl = last.getId().getUrl();
+                lastFetchedMethod = last.getId().getMethod().name();
+            }
+
             SampleDataToSTI sampleDataToSTI = new SampleDataToSTI();
-            sampleDataToSTI.setSampleDataToSTI(sampleData);
+            sampleDataToSTI.setSampleDataToSTI(sampleDataList);
             Map<String,Map<String, Map<Integer, List<SingleTypeInfo>>>> stiList = sampleDataToSTI.getSingleTypeInfoMap();
             OpenAPI openAPI = Main.init(apiCollection.getDisplayName(),stiList, includeHeaders, host);
             openAPIString = Main.convertOpenApiToJSON(openAPI);
@@ -84,4 +97,20 @@ public void setServletResponse(HttpServletResponse response) {
     public void setIncludeHeaders(boolean includeHeaders) {
         this.includeHeaders = includeHeaders;
     }
+
+    public String getLastFetchedUrl() {
+        return lastFetchedUrl;
+    }
+
+    public void setLastFetchedUrl(String lastFetchedUrl) {
+        this.lastFetchedUrl = lastFetchedUrl;
+    }
+
+    public String getLastFetchedMethod() {
+        return lastFetchedMethod;
+    }
+
+    public void setLastFetchedMethod(String lastFetchedMethod) {
+        this.lastFetchedMethod = lastFetchedMethod;
+    }
 }
diff --git a/apps/dashboard/src/main/java/com/akto/action/observe/InventoryAction.java b/apps/dashboard/src/main/java/com/akto/action/observe/InventoryAction.java
@@ -7,6 +7,8 @@
 import com.akto.dto.ApiInfo.ApiInfoKey;
 import com.akto.dto.type.SingleTypeInfo;
 import com.akto.dto.type.URLMethods.Method;
+import com.akto.log.LoggerMaker;
+import com.akto.log.LoggerMaker.LogDb;
 import com.akto.util.Constants;
 import com.mongodb.BasicDBList;
 import com.mongodb.BasicDBObject;
@@ -40,6 +42,7 @@ public class InventoryAction extends UserAction {
     // }
 
     public final static int DELTA_PERIOD_VALUE = 60 * 24 * 60 * 60;
+    private static final LoggerMaker loggerMaker = new LoggerMaker(InventoryAction.class);
 
     private String subType;
     public List<SingleTypeInfo> fetchSensitiveParams() {
@@ -579,6 +582,7 @@ private Bson prepareFilters() {
             }
         }
 
+        loggerMaker.infoAndAddToDb(filterList.toString(), LogDb.DASHBOARD);
         return Filters.and(filterList);
 
     }
@@ -607,6 +611,7 @@ private List<SingleTypeInfo> getMongoResults() {
 
         Bson sort = sortOrder == 1 ? Sorts.ascending(sortFields) : Sorts.descending(sortFields);
 
+        loggerMaker.infoAndAddToDb(String.format("skip: %s, limit: %s, sort: %s", skip, limit, sort), LogDb.DASHBOARD);
         List<SingleTypeInfo> list = SingleTypeInfoDao.instance.findAll(Filters.and(prepareFilters()), skip, limit, sort);
         return list;        
     }
@@ -617,7 +622,14 @@ private long getTotalParams() {
 
     public String fetchChanges() {
         response = new BasicDBObject();
-        response.put("data", new BasicDBObject("endpoints", getMongoResults()).append("total", getTotalParams()));
+
+        long totalParams = getTotalParams();
+        loggerMaker.infoAndAddToDb("Total params: " + totalParams, LogDb.DASHBOARD);
+
+        List<SingleTypeInfo> singleTypeInfos = getMongoResults();
+        loggerMaker.infoAndAddToDb("STI count: " + singleTypeInfos.size(), LogDb.DASHBOARD);
+
+        response.put("data", new BasicDBObject("endpoints", singleTypeInfos ).append("total", totalParams));
 
         return Action.SUCCESS.toUpperCase();
     }

diff --git a/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java b/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java
@@ -256,7 +256,13 @@ static void executePIISourceFetch() {
 
                     if (!dt.getBoolean("active", true)) {
                         PIISourceDao.instance.updateOne(findQ, Updates.unset("mapNameToPIIType." + piiKey));
-                        CustomDataTypeDao.instance.updateOne("name", piiKey, Updates.set("active", false));
+                        CustomDataType existingCDT = CustomDataTypeDao.instance.findOne("name", piiKey);
+                        if (existingCDT == null) {
+                            CustomDataTypeDao.instance.insertOne(getCustomDataTypeFromPiiType(piiSource, piiType, false));
+                            continue;
+                        } else {
+                            CustomDataTypeDao.instance.updateOne("name", piiKey, Updates.set("active", false));
+                        }
                     }
 
                     if (currTypes.containsKey(piiKey) && currTypes.get(piiKey).equals(piiType)) {
@@ -798,6 +804,13 @@ public void runInitializerFunctions() {
                 PIISourceDao.instance.insertOne(piiSource);
             }
 
+            if (PIISourceDao.instance.findOne("_id", "Fin") == null) {
+                String fileUrl = "https://raw.githubusercontent.com/akto-api-security/akto/master/pii-types/fintech.json";
+                PIISource piiSource = new PIISource(fileUrl, 0, 1638571050, 0, new HashMap<>(), true);
+                piiSource.setId("Fin");
+                PIISourceDao.instance.insertOne(piiSource);
+            }
+
             setUpWeeklyScheduler();
             setUpDailyScheduler();
             setUpWebhookScheduler();

diff --git a/apps/dashboard/src/test/java/com/akto/listener/TestFintechTypes.java b/apps/dashboard/src/test/java/com/akto/listener/TestFintechTypes.java
@@ -0,0 +1,44 @@
+package com.akto.listener;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.util.HashMap;
+
+import com.akto.MongoBasedTest;
+import com.akto.dao.CustomDataTypeDao;
+import com.akto.dao.context.Context;
+import com.akto.dao.pii.PIISourceDao;
+import com.akto.dto.CustomDataType;
+import com.akto.dto.pii.PIISource;
+import com.mongodb.BasicDBObject;
+
+import org.junit.Test;
+
+public class TestFintechTypes extends MongoBasedTest {
+
+
+    @Test
+    public void testTypes() {
+        String fileUrl = "https://raw.githubusercontent.com/akto-api-security/akto/master/pii-types/fintech.json";
+        PIISource piiSource = new PIISource(fileUrl, 0, 1638571050, 0, new HashMap<>(), true);
+        piiSource.setId("Fin");
+        PIISourceDao.instance.insertOne(piiSource);
+        InitializerListener.executePIISourceFetch();
+        Context.accountId.set(ACCOUNT_ID);
+        for(CustomDataType cdt: CustomDataTypeDao.instance.findAll(new BasicDBObject())) {
+            switch (cdt.getName().toUpperCase()) {
+                case "PAN CARD":
+                    assertTrue(cdt.validate("ABCDE9458J", "foo"));
+                    assertFalse(cdt.validate("ACDE9458J", "foo"));
+                    break;
+
+                default:
+                    break;
+            }
+
+        }
+    }
+
+}
diff --git a/apps/dashboard/web/src/apps/dashboard/views/marketplace/PageMarketplace.vue b/apps/dashboard/web/src/apps/dashboard/views/marketplace/PageMarketplace.vue
@@ -230,7 +230,7 @@ export default {
         let aktoTestTypes = await issuesApi.fetchAllSubCategories()
         this.businessCategories = aktoTestTypes.categories
         this.businessSubCategories = aktoTestTypes.subCategories
-        this.$router.push(this.leftNavItems[0].items[0].link)
+        this.$router.push(this.leftNavItems[0].items[0].link)        
     },
     computed: {
         ...mapState('marketplace', ['defaultSubcategories', 'userSubcategories', 'loading']),
@@ -338,4 +338,4 @@ export default {
 .c-item--title:hover {
     background: #f6f6f6;
 }
-</style>
+</style>
diff --git a/apps/dashboard/web/src/apps/dashboard/views/observe/inventory/api.js b/apps/dashboard/web/src/apps/dashboard/views/observe/inventory/api.js
@@ -39,12 +39,12 @@ export default {
             }
         })
     },
-    downloadOpenApiFile(apiCollectionId) {
+    downloadOpenApiFile(apiCollectionId,lastFetchedUrl, lastFetchedMethod) {
         return request({
             url: '/api/generateOpenApiFile',
             method: 'post',
             data: {
-                apiCollectionId
+                apiCollectionId, lastFetchedUrl, lastFetchedMethod
             }
         })
     },

diff --git a/apps/dashboard/web/src/apps/dashboard/views/observe/inventory/components/APIEndpoints.vue b/apps/dashboard/web/src/apps/dashboard/views/observe/inventory/components/APIEndpoints.vue
@@ -383,19 +383,31 @@ export default {
             }
         },
         async downloadOpenApiFile() {
-          var result = await this.$store.dispatch('inventory/downloadOpenApiFile')
-          let openApiString = result["openAPIString"]
-          var blob = new Blob([openApiString], {
-            type: "application/json",
-          });
-          const fileName = "open_api_" +this.apiCollectionName+ ".json";
-          saveAs(blob, fileName);
+          let lastFetchedUrl = null;
+          let lastFetchedMethod = null;
+          for (let index =0; index < 10; index++) {
+                var result = await this.$store.dispatch('inventory/downloadOpenApiFile', {lastFetchedUrl, lastFetchedMethod})
+                let openApiString = result["openAPIString"]
+                var blob = new Blob([openApiString], {
+                    type: "application/json",
+                });
+                const fileName = "open_api_" +this.apiCollectionName+ ".json";
+                saveAs(blob, fileName);
+
+                lastFetchedUrl = result["lastFetchedUrl"]
+                lastFetchedMethod = result["lastFetchedMethod"]
+
+                if (!lastFetchedUrl || !lastFetchedMethod) break;
+          }
+
+
           window._AKTO.$emit('SHOW_SNACKBAR', {
             show: true,
-            text: fileName + " downloaded !",
+            text: "OpenAPI spec file downloaded !",
             color: 'green'
           })
         },
+
         async exportToPostman() {
           var result = await this.$store.dispatch('inventory/exportToPostman')
           window._AKTO.$emit('SHOW_SNACKBAR', {

diff --git a/apps/dashboard/web/src/apps/dashboard/views/observe/inventory/store/module.js b/apps/dashboard/web/src/apps/dashboard/views/observe/inventory/store/module.js
@@ -181,8 +181,8 @@ const inventory = {
                 return resp
             })
         },
-        downloadOpenApiFile({commit,state}) {
-            return api.downloadOpenApiFile(state.apiCollectionId).then(resp => {
+        downloadOpenApiFile({commit,state}, {lastFetchedUrl, lastFetchedMethod}) {
+            return api.downloadOpenApiFile(state.apiCollectionId, lastFetchedUrl, lastFetchedMethod).then(resp => {
                 return resp
             })
         },

diff --git a/apps/dashboard/web/src/apps/dashboard/views/observe/sensitive/SensitiveData.vue b/apps/dashboard/web/src/apps/dashboard/views/observe/sensitive/SensitiveData.vue
@@ -210,10 +210,10 @@ export default {
     },
     methods: {
         async fetchRecentParamsForRequest(sortKey, sortOrder, skip, limit, filters, filterOperators) {
-            return await api.fetchChanges(sortKey, sortOrder, skip, limit, filters, filterOperators, this.startTimestamp, this.endTimestamp, true, true)
+            return await api.fetchChanges(sortKey, sortOrder, skip, limit, filters, filterOperators, 0, this.endTimestamp, true, true)
         },
         async fetchRecentParamsForResponse(sortKey, sortOrder, skip, limit, filters, filterOperators) {
-            return await api.fetchChanges(sortKey, sortOrder, skip, limit, filters, filterOperators, this.startTimestamp, this.endTimestamp, true, false)
+            return await api.fetchChanges(sortKey, sortOrder, skip, limit, filters, filterOperators, 0, this.endTimestamp, true, false)
         },
         ignoreForThisAPI(item) {
             this.ignoredCollection = item.name