Merge branch 'trunk' into configkey_is_not_well_formed

.github/workflows/codeql.yml

@@ -50,7 +50,7 @@ jobs:
 
       # Initializes the CodeQL tools for scanning.
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@fdcae64e1484d349b3366718cdfef3d404390e85 # v2.22.1
+        uses: github/codeql-action/init@d90b8d79de6dc1f58e83a1499aa58d6c93dc28de # v2.22.2
         with:
           languages: ${{ matrix.language }}
           # If you wish to specify custom queries, you can do so here or in a config file.
@@ -60,7 +60,7 @@ jobs:
       # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
       # If this step fails, then you should remove it and run the build manually (see below)
       - name: Autobuild
-        uses: github/codeql-action/autobuild@fdcae64e1484d349b3366718cdfef3d404390e85 # v2.22.1
+        uses: github/codeql-action/autobuild@d90b8d79de6dc1f58e83a1499aa58d6c93dc28de # v2.22.2
 
       # ℹ️ Command-line programs to run using the OS shell.
       # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
@@ -73,6 +73,6 @@ jobs:
       #   ./location_of_script_within_repo/buildscript.sh
 
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@fdcae64e1484d349b3366718cdfef3d404390e85 # v2.22.1
+        uses: github/codeql-action/analyze@d90b8d79de6dc1f58e83a1499aa58d6c93dc28de # v2.22.2
         with:
           category: "/language:${{matrix.language}}"

.github/workflows/scorecards.yml

@@ -67,6 +67,6 @@ jobs:
 
       # Upload the results to GitHub's code scanning dashboard.
       - name: "Upload to code-scanning"
-        uses: github/codeql-action/upload-sarif@fdcae64e1484d349b3366718cdfef3d404390e85 # v2.13.4
+        uses: github/codeql-action/upload-sarif@d90b8d79de6dc1f58e83a1499aa58d6c93dc28de # v2.13.4
         with:
           sarif_file: results.sarif

packages/at_persistence_secondary_server/CHANGELOG.md

@@ -1,3 +1,5 @@
+## 3.0.58
+- fix: Modify "lastCommittedSequenceNumberWithRegex" to return highest commitId among enrolled namespaces
 ## 3.0.57
 - fix: Refactor commit log keystore to optimize memory usage
 ## 3.0.56

packages/at_persistence_secondary_server/lib/src/compaction/at_compaction_stats_service_impl.dart

@@ -38,13 +38,13 @@ class AtCompactionStatsServiceImpl implements AtCompactionStatsService {
   ///changes the value of [compactionStatsKey] to match the AtLogType being processed
   void _getKey() {
     if (_atCompaction is AtCommitLog) {
-      compactionStatsKey = commitLogCompactionKey;
+      compactionStatsKey = AtConstants.commitLogCompactionKey;
     }
     if (_atCompaction is AtAccessLog) {
-      compactionStatsKey = accessLogCompactionKey;
+      compactionStatsKey = AtConstants.accessLogCompactionKey;
     }
     if (_atCompaction is AtNotificationKeystore) {
-      compactionStatsKey = notificationCompactionKey;
+      compactionStatsKey = AtConstants.notificationCompactionKey;
     }
   }
 }

packages/at_persistence_secondary_server/lib/src/log/commitlog/at_commit_log.dart

@@ -87,8 +87,10 @@ class AtCommitLog extends BaseAtCommitLog {
 
   /// Returns the latest committed sequence number with regex
   @server
-  Future<int?> lastCommittedSequenceNumberWithRegex(String regex) async {
-    return await _commitLogKeyStore.lastCommittedSequenceNumberWithRegex(regex);
+  Future<int?> lastCommittedSequenceNumberWithRegex(String regex,
+      {List<String>? enrolledNamespace}) async {
+    return await _commitLogKeyStore.lastCommittedSequenceNumberWithRegex(regex,
+        enrolledNamespace: enrolledNamespace);
   }
 
   /// Returns the first committed sequence number

packages/at_persistence_secondary_server/lib/src/log/commitlog/commit_log_keystore.dart

@@ -58,9 +58,11 @@ class CommitLogKeyStore extends BaseCommitLogKeyStore {
   }
 
   /// Returns the latest committed sequence number with regex
-  Future<int?> lastCommittedSequenceNumberWithRegex(String regex) async {
+  Future<int?> lastCommittedSequenceNumberWithRegex(String regex,
+      {List<String>? enrolledNamespace}) async {
     var lastCommittedEntry = (getBox() as Box).values.lastWhere(
-        (entry) => (_acceptKey(entry.atKey, regex)),
+        (entry) => (_acceptKey(entry.atKey, regex,
+            enrolledNamespace: enrolledNamespace)),
         orElse: () => NullCommitEntry());
     var lastCommittedSequenceNum =
         (lastCommittedEntry != null) ? lastCommittedEntry.key : null;
@@ -177,19 +179,41 @@ class CommitLogKeyStore extends BaseCommitLogKeyStore {
     }
   }
 
-  bool _acceptKey(String atKey, String regex) {
-    return _isRegexMatches(atKey, regex) || _isSpecialKey(atKey);
+  bool _acceptKey(String atKey, String regex,
+      {List<String>? enrolledNamespace}) {
+    return _isNamespaceAuthorised(atKey, enrolledNamespace) &&
+        (_isRegexMatches(atKey, regex) || _isSpecialKey(atKey));
+  }
+
+  bool _isNamespaceAuthorised(String atKey, List<String>? enrolledNamespace) {
+    // This is work-around for : https://github.com/atsign-foundation/at_server/issues/1570
+    if (atKey.toLowerCase() == 'configkey') {
+      return true;
+    }
+    String? keyNamespace = AtKey.fromString(atKey).namespace;
+    // If enrolledNamespace is null or keyNamespace is null, fallback to
+    // existing behaviour - the key is authorized for the client to receive. So return true.
+    if (enrolledNamespace == null ||
+        enrolledNamespace.isEmpty ||
+        (keyNamespace == null || keyNamespace.isEmpty)) {
+      return true;
+    }
+    if (enrolledNamespace.contains('*') ||
+        enrolledNamespace.contains(keyNamespace)) {
+      return true;
+    }
+    return false;
   }
 
   bool _isRegexMatches(String atKey, String regex) {
     return RegExp(regex).hasMatch(atKey);
   }
 
   bool _isSpecialKey(String atKey) {
-    return atKey.contains(AT_ENCRYPTION_SHARED_KEY) ||
+    return atKey.contains(AtConstants.atEncryptionSharedKey) ||
         atKey.startsWith('public:') ||
-        atKey.contains(AT_PKAM_SIGNATURE) ||
-        atKey.contains(AT_SIGNING_PRIVATE_KEY);
+        atKey.contains(AtConstants.atPkamSignature) ||
+        atKey.contains(AtConstants.atSigningPrivateKey);
   }
 
   /// Returns the latest commitEntry of the key.

packages/at_persistence_secondary_server/lib/src/model/at_meta_data.dart

@@ -136,21 +136,21 @@ class AtMetaData extends HiveObject {
     map['refreshAt'] = refreshAt?.toUtc().toString();
     map['status'] = status;
     map['version'] = version;
-    map[AT_TTL] = ttl;
-    map[AT_TTB] = ttb;
-    map[AT_TTR] = ttr;
-    map[CCD] = isCascade;
-    map[IS_BINARY] = isBinary;
-    map[IS_ENCRYPTED] = isEncrypted;
-    map[PUBLIC_DATA_SIGNATURE] = dataSignature;
-    map[SHARED_KEY_ENCRYPTED] = sharedKeyEnc;
-    map[SHARED_WITH_PUBLIC_KEY_CHECK_SUM] = pubKeyCS;
-    map[ENCODING] = encoding;
-    map[ENCRYPTING_KEY_NAME] = encKeyName;
-    map[ENCRYPTING_ALGO] = encAlgo;
-    map[IV_OR_NONCE] = ivNonce;
-    map[SHARED_KEY_ENCRYPTED_ENCRYPTING_KEY_NAME] = skeEncKeyName;
-    map[SHARED_KEY_ENCRYPTED_ENCRYPTING_ALGO] = skeEncAlgo;
+    map[AtConstants.ttl] = ttl;
+    map[AtConstants.ttb] = ttb;
+    map[AtConstants.ttr] = ttr;
+    map[AtConstants.ccd] = isCascade;
+    map[AtConstants.isBinary] = isBinary;
+    map[AtConstants.isEncrypted] = isEncrypted;
+    map[AtConstants.publicDataSignature] = dataSignature;
+    map[AtConstants.sharedKeyEncrypted] = sharedKeyEnc;
+    map[AtConstants.sharedWithPublicKeyCheckSum] = pubKeyCS;
+    map[AtConstants.encoding] = encoding;
+    map[AtConstants.encryptingKeyName] = encKeyName;
+    map[AtConstants.encryptingAlgo] = encAlgo;
+    map[AtConstants.ivOrNonce] = ivNonce;
+    map[AtConstants.sharedKeyEncryptedEncryptingKeyName] = skeEncKeyName;
+    map[AtConstants.sharedKeyEncryptedEncryptingAlgo] = skeEncAlgo;
     return map;
   }
 
@@ -178,33 +178,33 @@ class AtMetaData extends HiveObject {
         : (json['version'] == null)
             ? 0
             : json['version'];
-    ttl = (json[AT_TTL] is String)
-        ? int.parse(json[AT_TTL])
-        : (json[AT_TTL] == null)
+    ttl = (json[AtConstants.ttl] is String)
+        ? int.parse(json[AtConstants.ttl])
+        : (json[AtConstants.ttl] == null)
             ? null
-            : json[AT_TTL];
-    ttb = (json[AT_TTB] is String)
-        ? int.parse(json[AT_TTB])
-        : (json[AT_TTB] == null)
+            : json[AtConstants.ttl];
+    ttb = (json[AtConstants.ttb] is String)
+        ? int.parse(json[AtConstants.ttb])
+        : (json[AtConstants.ttb] == null)
             ? null
-            : json[AT_TTB];
-    ttr = (json[AT_TTR] is String)
-        ? int.parse(json[AT_TTR])
-        : (json[AT_TTR] == null)
+            : json[AtConstants.ttb];
+    ttr = (json[AtConstants.ttr] is String)
+        ? int.parse(json[AtConstants.ttr])
+        : (json[AtConstants.ttr] == null)
             ? null
-            : json[AT_TTR];
-    isCascade = json[CCD];
-    isBinary = json[IS_BINARY];
-    isEncrypted = json[IS_ENCRYPTED];
-    dataSignature = json[PUBLIC_DATA_SIGNATURE];
-    sharedKeyEnc = json[SHARED_KEY_ENCRYPTED];
-    pubKeyCS = json[SHARED_WITH_PUBLIC_KEY_CHECK_SUM];
-    encoding = json[ENCODING];
-    encKeyName = json[ENCRYPTING_KEY_NAME];
-    encAlgo = json[ENCRYPTING_ALGO];
-    ivNonce = json[IV_OR_NONCE];
-    skeEncKeyName = json[SHARED_KEY_ENCRYPTED_ENCRYPTING_KEY_NAME];
-    skeEncAlgo = json[SHARED_KEY_ENCRYPTED_ENCRYPTING_ALGO];
+            : json[AtConstants.ttr];
+    isCascade = json[AtConstants.ccd];
+    isBinary = json[AtConstants.isBinary];
+    isEncrypted = json[AtConstants.isEncrypted];
+    dataSignature = json[AtConstants.publicDataSignature];
+    sharedKeyEnc = json[AtConstants.sharedKeyEncrypted];
+    pubKeyCS = json[AtConstants.sharedWithPublicKeyCheckSum];
+    encoding = json[AtConstants.encoding];
+    encKeyName = json[AtConstants.encryptingKeyName];
+    encAlgo = json[AtConstants.encryptingAlgo];
+    ivNonce = json[AtConstants.ivOrNonce];
+    skeEncKeyName = json[AtConstants.sharedKeyEncryptedEncryptingKeyName];
+    skeEncAlgo = json[AtConstants.sharedKeyEncryptedEncryptingAlgo];
 
     return this;
   }

packages/at_persistence_secondary_server/pubspec.yaml

@@ -1,6 +1,6 @@
 name: at_persistence_secondary_server
 description: A Dart library with the implementation classes for the persistence layer of the secondary server.
-version: 3.0.57
+version: 3.0.58
 repository: https://github.com/atsign-foundation/at_server
 homepage: https://docs.atsign.com/
 

packages/at_persistence_secondary_server/test/at_compaction_stats_test.dart

@@ -76,7 +76,7 @@ Future<void> main() async {
       // Get Compaction Stats
       AtData? atData = await secondaryPersistenceStore!
           .getSecondaryKeyStore()
-          ?.get(at_commons.commitLogCompactionKey);
+          ?.get(at_commons.AtConstants.commitLogCompactionKey);
 
       // Assert Compaction Stats
       var decodedData = jsonDecode(atData!.data!) as Map;
@@ -110,7 +110,7 @@ Future<void> main() async {
       await atCompactionStatsServiceImpl.handleStats(atCompactionStats);
       AtData? atData = await secondaryPersistenceStore!
           .getSecondaryKeyStore()
-          ?.get(at_commons.accessLogCompactionKey);
+          ?.get(at_commons.AtConstants.accessLogCompactionKey);
       var data = (atData?.data);
       var decodedData = jsonDecode(data!) as Map;
       expect(decodedData["deletedKeysCount"], '3');

packages/at_persistence_secondary_server/test/commit_log_test.dart

@@ -133,10 +133,12 @@ void main() async {
         for (int i = 0; i < 10; i++) {
           if (i % 2 == 0) {
             await commitLogKeystore.getBox().add(CommitEntry(
-                'test_key_false_$i', CommitOp.UPDATE, DateTime.now()));
+                'test_key_false_$i.wavi@alice',
+                CommitOp.UPDATE,
+                DateTime.now()));
           } else {
             await commitLogKeystore.getBox().add(CommitEntry(
-                'test_key_false_$i', CommitOp.UPDATE, DateTime.now())
+                'test_key_false_$i.wavi@alice', CommitOp.UPDATE, DateTime.now())
               ..commitId = i);
           }
         }

packages/at_persistence_secondary_server/test/hive_keystore_impl_test.dart

@@ -238,7 +238,7 @@ void main() async {
       var keyStore = keyStoreManager.getSecondaryKeyStore()!;
       var atData = AtData();
       atData.data = '123';
-      final result = await keyStore.put(AT_PKAM_PRIVATE_KEY, atData);
+      final result = await keyStore.put(AtConstants.atPkamPrivateKey, atData);
       expect(result, isA<int>());
     });
 

packages/at_root_server/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 2.1.0
+- feat: rate limiting when looking up non-existent atSigns
+- feat: miscellaneous logging enhancements
+- fix: removed unnecessary 10-second delay during normal startup
 ## 2.0.5
 - upgrade persistence spec version
 ## 2.0.4

packages/at_root_server/Dockerfile

@@ -1,4 +1,4 @@
-FROM dart:3.1.3@sha256:b42328ca91e961fdec1346e930df42fb027d51cfba04af2a748764bbc867e1f8 AS buildimage
+FROM dart:3.1.3@sha256:1521c3d4e9e077a306e6af0857f49dd9465333cd0d140677c99a73c81582a63f AS buildimage
 ENV HOMEDIR=/atsign
 ENV BINARYDIR=/usr/local/at
 ENV USER_ID=1024

packages/at_root_server/lib/src/client/at_root_client.dart

@@ -5,13 +5,17 @@ import 'package:at_persistence_root_server/at_persistence_root_server.dart';
 import 'package:at_utils/at_logger.dart';
 import 'package:at_commons/at_commons.dart';
 
+enum RootClientState {listening,closing,closed}
+
 /// Represents Root Server client instance which contains socket on which a connection got established
 class RootClient {
   late Socket _socket;
   String? _address;
   int? _port;
   static final _keyStoreManager = KeystoreManagerImpl();
   final _buffer = StringBuffer(capacity: 255);
+  int notFoundCount = 0;
+  late RootClientState state;
 
   RootClient(Socket s) {
     _socket = s;
@@ -20,6 +24,7 @@ class RootClient {
     RootClientPool().add(this);
     _socket.listen(_messageHandler,
         onError: _errorHandler, onDone: _finishedHandler);
+    state = RootClientState.listening;
   }
 
   var logger = AtSignLogger('RootClient');
@@ -29,26 +34,43 @@ class RootClient {
   ///  @param - data : data received from client over the socket
   Future<void> _messageHandler(data) async {
     try {
-      logger.info('In root client _messagehandler');
+      logger.finest('${_address}:${_port} In root client _messagehandler');
       var message = utf8.decode(data);
       message = message.toLowerCase();
       _buffer.append(message);
-      if (_buffer.getData()!.trim() == '@exit') {
+      var lookupPayload = _buffer.getData()!.trim();
+      if (lookupPayload == '@exit') {
         _finishedHandler();
         return;
       } else {
         if (_buffer.isEnd()) {
           var result = await _keyStoreManager
               .getKeyStore()
-              .get(_buffer.getData()!.trim());
-          logger.finer('result:${result}');
+              .get(lookupPayload);
+          logger.info('${_address}:${_port} Looked up: $lookupPayload | Found: $result');
+
+          if (result == null) {
+            notFoundCount++;
+            if (notFoundCount > 1) {
+              // If we've already had at least one lookup of a non-existent atSign
+              // then we will wait a few seconds before responding
+              await Future.delayed(Duration(seconds: 3));
+            }
+
+            if (notFoundCount > 2) {
+              // And if we've already had at least two lookups of a non-existent
+              // atSign then we will terminate the connection
+              _finishedHandler();
+              return;
+            }
+          }
           result ??= 'null';
           write(result + '\r\n@');
           _buffer.clear();
         }
       }
     } on Exception catch (exception) {
-      logger.severe(exception);
+      logger.severe('${_address}:${_port} _messageHandler | $exception');
       _socket.destroy();
     } catch (error) {
       _errorHandler(error.toString());
@@ -59,16 +81,19 @@ class RootClient {
   ///  Return type - void
   ///  @param - error : error string
   void _errorHandler(error) {
-    logger.severe('${_address}:${_port} Error: $error');
-    removeClient(this);
-    _socket.destroy();
+    if (state == RootClientState.listening) {
+      logger.severe('${_address}:${_port} Error: $error');
+      removeClient(this);
+    }
   }
 
   /// _finishedHandler close the client connection and remove from client pool
   ///  Return type - void
   void _finishedHandler() {
-    logger.info('${_address}:${_port} Disconnected');
-    removeClient(this);
+    if (state == RootClientState.listening) {
+      logger.info('${_address}:${_port} Disconnected');
+      removeClient(this);
+    }
   }
 
   /// write - Writes received message to the client socket
@@ -83,7 +108,9 @@ class RootClient {
   /// return type - void
   /// @param - rootClient - Instance of RootClient
   void removeClient(RootClient rootClient) {
+    state = RootClientState.closing;
     rootClient._socket.destroy();
     RootClientPool().remove(rootClient);
+    state = RootClientState.closed;
   }
 }