Skip to content

Commit

Permalink
Merge branch 'trunk' into 2121-uptake-pubkeyhash-changes
Browse files Browse the repository at this point in the history
  • Loading branch information
sitaram-kalluri authored Dec 5, 2024
2 parents 24194dc + 1581a0b commit c0924fa
Show file tree
Hide file tree
Showing 35 changed files with 1,199 additions and 145 deletions.
14 changes: 7 additions & 7 deletions .github/workflows/at_server.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ jobs:
steps:
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2

- uses: dart-lang/setup-dart@0a8a0fc875eb934c15d08629302413c671d3f672 # v1.6.5
- uses: dart-lang/setup-dart@e630b99d28a3b71860378cafdc2a067c71107f94 # v1.7.0
with:
sdk: ${{ matrix.dart-channel}}

Expand Down Expand Up @@ -179,7 +179,7 @@ jobs:
run: |
echo "hashes=$(cat checksums.txt | base64 -w0)" >> "$GITHUB_OUTPUT"
- if: ${{ matrix.dart-channel == 'stable' && startsWith(github.ref, 'refs/tags/') }}
uses: actions/attest-build-provenance@ef244123eb79f2f7a7e75d99086184180e6d0018 # v1.4.4
uses: actions/attest-build-provenance@619dbb2e03e0189af0c55118e7d3c5e129e99726 # v2.0.0
with:
subject-path: 'sboms/**'

Expand Down Expand Up @@ -218,7 +218,7 @@ jobs:
steps:
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2

- uses: dart-lang/setup-dart@0a8a0fc875eb934c15d08629302413c671d3f672 # v1.6.5
- uses: dart-lang/setup-dart@e630b99d28a3b71860378cafdc2a067c71107f94 # v1.7.0
with:
sdk: ${{ matrix.dart-channel}}

Expand Down Expand Up @@ -356,7 +356,7 @@ jobs:

steps:
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- uses: dart-lang/setup-dart@0a8a0fc875eb934c15d08629302413c671d3f672 # v 1.6.5
- uses: dart-lang/setup-dart@e630b99d28a3b71860378cafdc2a067c71107f94 # v 1.7.0
with:
sdk: stable

Expand Down Expand Up @@ -385,7 +385,7 @@ jobs:

steps:
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- uses: dart-lang/setup-dart@0a8a0fc875eb934c15d08629302413c671d3f672 # v1.6.5
- uses: dart-lang/setup-dart@e630b99d28a3b71860378cafdc2a067c71107f94 # v1.7.0
with:
sdk: stable

Expand Down Expand Up @@ -414,7 +414,7 @@ jobs:

steps:
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- uses: dart-lang/setup-dart@0a8a0fc875eb934c15d08629302413c671d3f672 # v1.6.5
- uses: dart-lang/setup-dart@e630b99d28a3b71860378cafdc2a067c71107f94 # v1.7.0
with:
sdk: stable

Expand Down Expand Up @@ -467,7 +467,7 @@ jobs:
fi
- name: Install Dart
uses: dart-lang/setup-dart@0a8a0fc875eb934c15d08629302413c671d3f672 # v1.6.5
uses: dart-lang/setup-dart@e630b99d28a3b71860378cafdc2a067c71107f94 # v1.7.0
with:
sdk: stable

Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/at_server_dev_deploy.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ jobs:
uses: google-github-actions/setup-gcloud@6189d56e4096ee891640bb02ac264be376592d6a # v2.1.2

# Get the GKE credentials so we can deploy to the cluster
- uses: google-github-actions/get-gke-credentials@206d64b64b0eba0a6e2f25113d044c31776ca8d6 # v2.2.2
- uses: google-github-actions/get-gke-credentials@9025e8f90f2d8e0c3dafc3128cc705a26d992a6a # v2.3.0
with:
cluster_name: ${{ secrets.GKE_DEV_CLUSTER }}
location: ${{ secrets.GKE_DEV_ZONE }}
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/at_server_prod_deploy.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@ jobs:
uses: google-github-actions/setup-gcloud@6189d56e4096ee891640bb02ac264be376592d6a # v2.1.2

# Get the GKE credentials so we can deploy to the cluster
- uses: google-github-actions/get-gke-credentials@206d64b64b0eba0a6e2f25113d044c31776ca8d6 # v2.2.2
- uses: google-github-actions/get-gke-credentials@9025e8f90f2d8e0c3dafc3128cc705a26d992a6a # v2.3.0
with:
cluster_name: ${{ secrets.GKE_PROD_CLUSTER }}
location: ${{ secrets.GKE_PROD_ZONE }}
Expand Down
6 changes: 3 additions & 3 deletions .github/workflows/codeql.yml
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ jobs:

# Initializes the CodeQL tools for scanning.
- name: Initialize CodeQL
uses: github/codeql-action/init@f09c1c0a94de965c15400f5634aa42fac8fb8f88 # v3.27.5
uses: github/codeql-action/init@aa578102511db1f4524ed59b8cc2bae4f6e88195 # v3.27.6
with:
languages: ${{ matrix.language }}
# If you wish to specify custom queries, you can do so here or in a config file.
Expand All @@ -60,7 +60,7 @@ jobs:
# Autobuild attempts to build any compiled languages (C/C++, C#, or Java).
# If this step fails, then you should remove it and run the build manually (see below)
- name: Autobuild
uses: github/codeql-action/autobuild@f09c1c0a94de965c15400f5634aa42fac8fb8f88 # v3.27.5
uses: github/codeql-action/autobuild@aa578102511db1f4524ed59b8cc2bae4f6e88195 # v3.27.6

# ℹ️ Command-line programs to run using the OS shell.
# 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
Expand All @@ -73,6 +73,6 @@ jobs:
# ./location_of_script_within_repo/buildscript.sh

- name: Perform CodeQL Analysis
uses: github/codeql-action/analyze@f09c1c0a94de965c15400f5634aa42fac8fb8f88 # v3.27.5
uses: github/codeql-action/analyze@aa578102511db1f4524ed59b8cc2bae4f6e88195 # v3.27.6
with:
category: "/language:${{matrix.language}}"
2 changes: 1 addition & 1 deletion .github/workflows/melos_bootstrap.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- uses: subosito/flutter-action@74af56c5ed2697ba4621264652728e8d217e53d3 # v2.17.0
- uses: subosito/flutter-action@f2c4f6686ca8e8d6e6d0f28410eeef506ed66aff # v2.18.0
with:
channel: "stable"
- name: flutter pub get
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/promote_canary.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -164,7 +164,7 @@ jobs:
working-directory: sboms
run: |
echo "hashes=$(cat checksums.txt | base64 -w0)" >> "$GITHUB_OUTPUT"
- uses: actions/attest-build-provenance@ef244123eb79f2f7a7e75d99086184180e6d0018 # v1.4.4
- uses: actions/attest-build-provenance@619dbb2e03e0189af0c55118e7d3c5e129e99726 # v2.0.0
with:
subject-path: 'sboms/**'

Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/scorecards.yml
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,6 @@ jobs:

# Upload the results to GitHub's code scanning dashboard.
- name: "Upload to code-scanning"
uses: github/codeql-action/upload-sarif@f09c1c0a94de965c15400f5634aa42fac8fb8f88 # v3.27.5
uses: github/codeql-action/upload-sarif@aa578102511db1f4524ed59b8cc2bae4f6e88195 # v3.27.6
with:
sarif_file: results.sarif
10 changes: 10 additions & 0 deletions packages/at_persistence_secondary_server/CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,3 +1,13 @@
## 3.1.0
- feat: commit log changes for sync skipDeletesUntil feature
- build[deps]: Upgraded the following package:
- at_commons to v5.1.0
## 3.0.66
- feat: Add "PublicKeyHash" to the "AtMetadata" which holds the hash value of encryption public key
- build[deps]: Upgraded the following packages:
- at_commons to v5.0.2
- lints to v5.0.0
- test to v1.25.8
## 3.0.65
- fix: Modified checks in commit log keystore _alwaysIncludeInSync method to match only reserved shared_key,
encryption public key and public key without namespace.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,8 @@ class AtConfig {
persistenceManager = SecondaryPersistenceStoreFactory.getInstance()
.getSecondaryPersistenceStore(_atSign)!
.getHivePersistenceManager()!;
configKey = HiveKeyStoreHelper.getInstance().prepareKey('private:blocklist$_atSign');
configKey = HiveKeyStoreHelper.getInstance()
.prepareKey('private:blocklist$_atSign');
}

///Returns 'success' on adding unique [blockList] into blocklist.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,9 @@ class HivePersistenceManager with HiveBase {
if (!Hive.isAdapterRegistered(AtMetaDataAdapter().typeId)) {
Hive.registerAdapter(AtMetaDataAdapter());
}
if (!Hive.isAdapterRegistered(PublicKeyHashAdapater().typeId)) {
Hive.registerAdapter(PublicKeyHashAdapater());
}

var secret = await _getHiveSecretFromFile(_atsign!, storagePath);
_boxName = AtUtils.getShaForAtSign(_atsign!);
Expand Down Expand Up @@ -113,10 +116,11 @@ class HivePersistenceManager with HiveBase {
}

//TODO change into to Duration and construct cron string dynamically
void scheduleKeyExpireTask(int? runFrequencyMins, {Duration? runTimeInterval, bool skipCommits = false}) {
void scheduleKeyExpireTask(int? runFrequencyMins,
{Duration? runTimeInterval, bool skipCommits = false}) {
logger.finest('scheduleKeyExpireTask starting cron job.');
Schedule schedule;
if(runTimeInterval != null){
if (runTimeInterval != null) {
schedule = Schedule(seconds: runTimeInterval.inSeconds);
} else {
schedule = Schedule.parse('*/$runFrequencyMins * * * *');
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -127,12 +127,13 @@ class AtCommitLog extends BaseAtCommitLog {
/// Returns the Iterator of [_commitLogCacheMap] from the commitId specified.
@server
Iterator<MapEntry<String, CommitEntry>> getEntries(int commitId,
{String? regex, int limit = 25}) {
{String? regex, int limit = 25, int? skipDeletesUntil}) {
// If regex is null or isEmpty set regex to match all keys
if (regex == null || regex.isEmpty) {
regex = '.*';
}
return _commitLogKeyStore.getEntries(commitId, regex: regex, limit: limit);
return _commitLogKeyStore.getEntries(commitId,
regex: regex, limit: limit, skipDeletesUntil: skipDeletesUntil);
}

Future<void> _publishChangeEvent(CommitEntry commitEntry) async {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,9 @@ import 'package:at_persistence_secondary_server/src/keystore/hive_base.dart';
import 'package:at_utils/at_utils.dart';
import 'package:hive/hive.dart';
import 'package:meta/meta.dart';
import 'package:at_persistence_secondary_server/src/log/commitlog/sync/fetch_all_keys_strategy.dart';
import 'package:at_persistence_secondary_server/src/log/commitlog/sync/skip_deletes_strategy.dart';
import 'package:at_persistence_secondary_server/src/log/commitlog/sync/sync_keys_fetch_strategy.dart';

@server
class CommitLogKeyStore extends BaseCommitLogKeyStore {
Expand All @@ -12,8 +15,11 @@ class CommitLogKeyStore extends BaseCommitLogKeyStore {

int get latestCommitId => commitLogCache.latestCommitId;

late SyncKeysFetchStrategy _syncKeysFetchStrategy;

CommitLogKeyStore(String currentAtSign) : super(currentAtSign) {
commitLogCache = CommitLogCache(this);
_syncKeysFetchStrategy = FetchAllKeysStrategy();
}

@override
Expand Down Expand Up @@ -184,51 +190,8 @@ class CommitLogKeyStore extends BaseCommitLogKeyStore {
/// if regex is passed, key has to match the regex or it has to be a special key.
bool _shouldIncludeKeyInSyncResponse(String atKey, String regex,
{List<String>? enrolledNamespace}) {
return _isNamespaceAuthorised(atKey, enrolledNamespace) &&
(_keyMatchesRegex(atKey, regex) || _alwaysIncludeInSync(atKey));
}

bool _isNamespaceAuthorised(
String atKeyAsString, List<String>? enrolledNamespace) {
// This is work-around for : https://github.com/atsign-foundation/at_server/issues/1570
if (atKeyAsString.toLowerCase() == 'configkey') {
return true;
}
late AtKey atKey;
try {
atKey = AtKey.fromString(atKeyAsString);
} on InvalidSyntaxException catch (_) {
_logger.warning(
'_isNamespaceAuthorized found an invalid key "$atKeyAsString" in the commit log. Returning false');
return false;
}
String? keyNamespace = atKey.namespace;
// If enrolledNamespace is null or keyNamespace is null, fallback to
// existing behaviour - the key is authorized for the client to receive. So return true.
if (enrolledNamespace == null ||
enrolledNamespace.isEmpty ||
(keyNamespace == null || keyNamespace.isEmpty)) {
return true;
}
if (enrolledNamespace.contains('*') ||
enrolledNamespace.contains(keyNamespace)) {
return true;
}
return false;
}

bool _keyMatchesRegex(String atKey, String regex) {
return RegExp(regex).hasMatch(atKey);
}

/// match keys which have to included in sync irrespective of whether regex matches
/// e.g @bob:shared_key@alice, shared_key.bob@alice, public:publickey@alice,
/// public:phone@alice (public key without namespace)
bool _alwaysIncludeInSync(String atKey) {
return (atKey.contains(AtConstants.atEncryptionSharedKey) &&
RegexUtil.keyType(atKey, false) == KeyType.reservedKey) ||
atKey.startsWith(AtConstants.atEncryptionPublicKey) ||
(atKey.startsWith('public:') && !atKey.contains('.'));
return _syncKeysFetchStrategy.shouldIncludeKeyInSyncResponse(atKey, regex,
enrolledNamespace: enrolledNamespace);
}

/// Returns the latest commitEntry of the key.
Expand All @@ -238,13 +201,16 @@ class CommitLogKeyStore extends BaseCommitLogKeyStore {

/// Returns the Iterator of entries as Key value pairs after the given the [commitId] for the keys that matches the [regex]
Iterator<MapEntry<String, CommitEntry>> getEntries(int commitId,
{String regex = '.*', int limit = 25}) {
{String regex = '.*', int limit = 25, int? skipDeletesUntil}) {
SyncKeysFetchStrategy syncKeysFetchStrategy = skipDeletesUntil != null
? SkipDeleteStrategy(skipDeletesUntil, commitLogCache.latestCommitId)
: _syncKeysFetchStrategy;
Iterable<MapEntry<String, CommitEntry>> commitEntriesIterable =
commitLogCache
.entriesList()
.where((element) =>
element.value.commitId! >= commitId &&
_shouldIncludeKeyInSyncResponse(element.value.atKey!, regex))
syncKeysFetchStrategy.shouldIncludeEntryInSyncResponse(
element.value, commitId, regex))
.take(limit);
return commitEntriesIterable.iterator;
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
import 'package:at_persistence_secondary_server/src/log/commitlog/commit_entry.dart';
import 'package:at_persistence_secondary_server/src/log/commitlog/sync/sync_keys_fetch_strategy.dart';

/// Returns the commit entries which have to be synced from server to client
class FetchAllKeysStrategy extends SyncKeysFetchStrategy {
@override
bool shouldIncludeEntryInSyncResponse(
CommitEntry commitEntry, int commitId, String regex,
{List<String>? enrolledNamespace}) {
return commitEntry.commitId! >= commitId &&
super.shouldIncludeKeyInSyncResponse(commitEntry.atKey!, regex,
enrolledNamespace: enrolledNamespace);
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
import 'package:at_persistence_secondary_server/src/log/commitlog/commit_entry.dart';
import 'package:at_persistence_secondary_server/src/log/commitlog/sync/sync_keys_fetch_strategy.dart';

/// Returns the commit entries to be returned in sync response from server to client except delete commit entries.
class SkipDeleteStrategy extends SyncKeysFetchStrategy {
late int skipDeletesUntil;
late int latestCommitId;
SkipDeleteStrategy(this.skipDeletesUntil, this.latestCommitId);
@override
bool shouldIncludeEntryInSyncResponse(
CommitEntry commitEntry, int commitId, String regex,
{List<String>? enrolledNamespace}) {
// do not include delete commit entries between commitId and skipDeletesUntil, except when delete is the last commit entry
if (commitEntry.operation == CommitOp.DELETE &&
commitEntry.commitId! <= skipDeletesUntil &&
commitEntry.commitId! >= commitId &&
commitEntry.commitId != latestCommitId) {
return false;
}
return commitEntry.commitId! >= commitId &&
super.shouldIncludeKeyInSyncResponse(commitEntry.atKey!, regex,
enrolledNamespace: enrolledNamespace);
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
import 'package:at_commons/at_commons.dart';
import 'package:at_persistence_secondary_server/at_persistence_secondary_server.dart';
import 'package:at_utils/at_utils.dart';

abstract class SyncKeysFetchStrategy {
final _logger = AtSignLogger('SyncKeysFetchStrategy');

/// Returns true if the commit entry should be included in sync response, false otherwise
bool shouldIncludeEntryInSyncResponse(
CommitEntry commitEntry, int commitId, String regex,
{List<String>? enrolledNamespace});

/// if enrolledNamespace is passed, key namespace should be in enrolledNamespace list and
/// atKey should match regex or should be a special key that is always included in sync.
bool shouldIncludeKeyInSyncResponse(String atKey, String regex,
{List<String>? enrolledNamespace}) {
return isNamespaceAuthorised(atKey, enrolledNamespace) &&
(keyMatchesRegex(atKey, regex) || alwaysIncludeInSync(atKey));
}

/// Returns true if atKey namespace is empty or null/ enrolledNamespace is empty or null
/// if enrolledNamespace contains atKey namespace, return true. false otherwise
bool isNamespaceAuthorised(
String atKeyAsString, List<String>? enrolledNamespace) {
// This is work-around for : https://github.com/atsign-foundation/at_server/issues/1570
if (atKeyAsString.toLowerCase() == 'configkey') {
return true;
}
late AtKey atKey;
try {
atKey = AtKey.fromString(atKeyAsString);
} on InvalidSyntaxException catch (_) {
_logger.warning(
'_isNamespaceAuthorized found an invalid key "$atKeyAsString" in the commit log. Returning false');
return false;
}
String? keyNamespace = atKey.namespace;
// If enrolledNamespace is null or keyNamespace is null, fallback to
// existing behaviour - the key is authorized for the client to receive. So return true.
if (enrolledNamespace == null ||
enrolledNamespace.isEmpty ||
(keyNamespace == null || keyNamespace.isEmpty)) {
return true;
}
if (enrolledNamespace.contains('*') ||
enrolledNamespace.contains(keyNamespace)) {
return true;
}
return false;
}

/// Returns true if atKey matches regex. false otherwise
bool keyMatchesRegex(String atKey, String regex) {
return RegExp(regex).hasMatch(atKey);
}

/// match keys which have to included in sync irrespective of whether regex matches
/// e.g @bob:shared_key@alice, shared_key.bob@alice, public:publickey@alice,
/// public:phone@alice (public key without namespace)
bool alwaysIncludeInSync(String atKey) {
return (atKey.contains(AtConstants.atEncryptionSharedKey) &&
RegexUtil.keyType(atKey, false) == KeyType.reservedKey) ||
atKey.startsWith(AtConstants.atEncryptionPublicKey) ||
(atKey.startsWith('public:') && !atKey.contains('.'));
}
}
Loading

0 comments on commit c0924fa

Please sign in to comment.