Skip to content

Commit

Permalink
Trigger deletion of objects of studies from storage system by exceed…
Browse files Browse the repository at this point in the history
…ing of configured disk usage threshold fix #4584

 Extend RESTful service to list configured Storage Systems to return configured Deleter Thresholds of Max Usable Space and current used disk space #4585
  • Loading branch information
gunterze committed Oct 21, 2024
1 parent 1a611d4 commit 5b72b6d
Show file tree
Hide file tree
Showing 13 changed files with 257 additions and 36 deletions.
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# Generated by Apache Directory Studio on September 25, 2024, 6:44:48 PM
# Generated by Apache Directory Studio on October 21, 2024, 5:22:28 PM

# SCHEMA "DCM4CHEE-ARCHIVE"
dn: cn=dcm4chee-archive, ou=schema
Expand Down Expand Up @@ -6457,6 +6457,35 @@ m-equality: booleanMatch
m-syntax: 1.3.6.1.4.1.1466.115.121.1.7
m-singleValue: TRUE

dn: m-oid=1.2.40.0.13.1.15.110.3.484, ou=attributetypes, cn=dcm4chee-archive, ou
=schema
objectclass: metaAttributeType
objectclass: metaTop
objectclass: top
m-oid: 1.2.40.0.13.1.15.110.3.484
m-name: dcmDeleterThresholdMaxUsableSpace
m-description: Maximal Usable Space on Storage System to trigger deletion. If pr
esent, studies are deleted from the Storage System configured for cache (dcmSto
rageDuration=CACHE) or temporary (dcmStorageDuration=TEMPORARY) storage, if the
used disk space exceeds that value. Format [nn"[Used(s)"<schedule>"]"]nnn(MB|G
B|MiB|GiB). Only effective if dcmDeleterThresholdBlocksFilePath is also specifi
ed.
m-equality: caseExactIA5Match
m-syntax: 1.3.6.1.4.1.1466.115.121.1.26

dn: m-oid=1.2.40.0.13.1.15.110.3.485, ou=attributetypes, cn=dcm4chee-archive, ou
=schema
objectclass: metaAttributeType
objectclass: metaTop
objectclass: top
m-oid: 1.2.40.0.13.1.15.110.3.485
m-name: dcmDeleterThresholdBlocksFilePath
m-description: Path of file containing the current used disk space in blocks (10
24 bytes), periodically updated by an external application.
m-equality: caseExactIA5Match
m-syntax: 1.3.6.1.4.1.1466.115.121.1.26
m-singleValue: TRUE

dn: ou=comparators, cn=dcm4chee-archive, ou=schema
objectclass: organizationalUnit
objectclass: top
Expand Down Expand Up @@ -6587,6 +6616,8 @@ m-may: dcmStorageThreshold
m-may: dcmStorageThresholdExceeded
m-may: dcmStorageThresholdExceedsPermanently
m-may: dcmDeleterThreshold
m-may: dcmDeleterThresholdMaxUsableSpace
m-may: dcmDeleterThresholdBlocksFilePath
m-may: dcmDeleteStudiesOlderThan
m-may: dcmDeleteStudiesReceivedBefore
m-may: dcmDeleteStudiesNotUsedSince
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2357,6 +2357,15 @@ attributeTypes: ( 1.2.40.0.13.1.15.110.3.483 NAME 'dcmSingleExportStorageByStudy
EQUALITY booleanMatch
SYNTAX 1.3.6.1.4.1.1466.115.121.1.7
SINGLE-VALUE )
attributeTypes: ( 1.2.40.0.13.1.15.110.3.484 NAME 'dcmDeleterThresholdMaxUsableSpace'
DESC 'Maximal Usable Space on Storage System to trigger deletion. If present, studies are deleted from the Storage System configured for cache (dcmStorageDuration=CACHE) or temporary (dcmStorageDuration=TEMPORARY) storage, if the used disk space exceeds that value. Format [nn"[Used(s)"<schedule>"]"]nnn(MB|GB|MiB|GiB). Only effective if dcmDeleterThresholdBlocksFilePath is also specified.'
EQUALITY caseExactIA5Match
SYNTAX 1.3.6.1.4.1.1466.115.121.1.26 )
attributeTypes: ( 1.2.40.0.13.1.15.110.3.485 NAME 'dcmDeleterThresholdBlocksFilePath'
DESC 'Path of file containing the current used disk space in blocks (1024 bytes), periodically updated by an external application.'
EQUALITY caseExactIA5Match
SYNTAX 1.3.6.1.4.1.1466.115.121.1.26
SINGLE-VALUE )
objectClasses: ( 1.2.40.0.13.1.15.110.4.4 NAME 'dcmArchiveDevice'
DESC 'DICOM Archive Device related information'
SUP top AUXILIARY
Expand Down Expand Up @@ -2863,6 +2872,8 @@ objectClasses: ( 1.2.40.0.13.1.15.110.4.12 NAME 'dcmStorage'
dcmStorageThresholdExceeded $
dcmStorageThresholdExceedsPermanently $
dcmDeleterThreshold $
dcmDeleterThresholdMaxUsableSpace $
dcmDeleterThresholdBlocksFilePath $
dcmDeleteStudiesOlderThan $
dcmDeleteStudiesReceivedBefore $
dcmDeleteStudiesNotUsedSince $
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2830,6 +2830,17 @@ attributetype ( 1.2.40.0.13.1.15.110.3.483 NAME 'dcmSingleExportStorageByStudy'
SYNTAX 1.3.6.1.4.1.1466.115.121.1.7
SINGLE-VALUE )

attributetype ( 1.2.40.0.13.1.15.110.3.484 NAME 'dcmDeleterThresholdMaxUsableSpace'
DESC 'Maximal Usable Space on Storage System to trigger deletion. If present, studies are deleted from the Storage System configured for cache (dcmStorageDuration=CACHE) or temporary (dcmStorageDuration=TEMPORARY) storage, if the used disk space exceeds that value. Format [nn"[Used(s)"<schedule>"]"]nnn(MB|GB|MiB|GiB). Only effective if dcmDeleterThresholdBlocksFilePath is also specified.'
EQUALITY caseExactIA5Match
SYNTAX 1.3.6.1.4.1.1466.115.121.1.26 )

attributetype ( 1.2.40.0.13.1.15.110.3.485 NAME 'dcmDeleterThresholdBlocksFilePath'
DESC 'Path of file containing the current used disk space in blocks (1024 bytes), periodically updated by an external application.'
EQUALITY caseExactIA5Match
SYNTAX 1.3.6.1.4.1.1466.115.121.1.26
SINGLE-VALUE )

objectclass ( 1.2.40.0.13.1.15.110.4.4 NAME 'dcmArchiveDevice'
DESC 'DICOM Archive Device related information'
SUP top AUXILIARY
Expand Down Expand Up @@ -3340,6 +3351,8 @@ objectclass ( 1.2.40.0.13.1.15.110.4.12 NAME 'dcmStorage'
dcmStorageThresholdExceeded $
dcmStorageThresholdExceedsPermanently $
dcmDeleterThreshold $
dcmDeleterThresholdMaxUsableSpace $
dcmDeleterThresholdBlocksFilePath $
dcmDeleteStudiesOlderThan $
dcmDeleteStudiesReceivedBefore $
dcmDeleteStudiesNotUsedSince $
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2357,6 +2357,15 @@ olcAttributeTypes: ( 1.2.40.0.13.1.15.110.3.483 NAME 'dcmSingleExportStorageBySt
EQUALITY booleanMatch
SYNTAX 1.3.6.1.4.1.1466.115.121.1.7
SINGLE-VALUE )
olcAttributeTypes: ( 1.2.40.0.13.1.15.110.3.484 NAME 'dcmDeleterThresholdMaxUsableSpace'
DESC 'Maximal Usable Space on Storage System to trigger deletion. If present, studies are deleted from the Storage System configured for cache (dcmStorageDuration=CACHE) or temporary (dcmStorageDuration=TEMPORARY) storage, if the used disk space exceeds that value. Format [nn"[Used(s)"<schedule>"]"]nnn(MB|GB|MiB|GiB). Only effective if dcmDeleterThresholdBlocksFilePath is also specified.'
EQUALITY caseExactIA5Match
SYNTAX 1.3.6.1.4.1.1466.115.121.1.26 )
olcAttributeTypes: ( 1.2.40.0.13.1.15.110.3.485 NAME 'dcmDeleterThresholdBlocksFilePath'
DESC 'Path of file containing the current used disk space in blocks (1024 bytes), periodically updated by an external application.'
EQUALITY caseExactIA5Match
SYNTAX 1.3.6.1.4.1.1466.115.121.1.26
SINGLE-VALUE )
-
replace: olcObjectClasses
olcObjectClasses: ( 1.2.40.0.13.1.15.110.4.4 NAME 'dcmArchiveDevice'
Expand Down Expand Up @@ -2865,6 +2874,8 @@ olcObjectClasses: ( 1.2.40.0.13.1.15.110.4.12 NAME 'dcmStorage'
dcmStorageThresholdExceeded $
dcmStorageThresholdExceedsPermanently $
dcmDeleterThreshold $
dcmDeleterThresholdMaxUsableSpace $
dcmDeleterThresholdBlocksFilePath $
dcmDeleteStudiesOlderThan $
dcmDeleteStudiesReceivedBefore $
dcmDeleteStudiesNotUsedSince $
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2356,6 +2356,15 @@ olcAttributeTypes: ( 1.2.40.0.13.1.15.110.3.483 NAME 'dcmSingleExportStorageBySt
EQUALITY booleanMatch
SYNTAX 1.3.6.1.4.1.1466.115.121.1.7
SINGLE-VALUE )
olcAttributeTypes: ( 1.2.40.0.13.1.15.110.3.484 NAME 'dcmDeleterThresholdMaxUsableSpace'
DESC 'Maximal Usable Space on Storage System to trigger deletion. If present, studies are deleted from the Storage System configured for cache (dcmStorageDuration=CACHE) or temporary (dcmStorageDuration=TEMPORARY) storage, if the used disk space exceeds that value. Format [nn"[Used(s)"<schedule>"]"]nnn(MB|GB|MiB|GiB). Only effective if dcmDeleterThresholdBlocksFilePath is also specified.'
EQUALITY caseExactIA5Match
SYNTAX 1.3.6.1.4.1.1466.115.121.1.26 )
olcAttributeTypes: ( 1.2.40.0.13.1.15.110.3.485 NAME 'dcmDeleterThresholdBlocksFilePath'
DESC 'Path of file containing the current used disk space in blocks (1024 bytes), periodically updated by an external application.'
EQUALITY caseExactIA5Match
SYNTAX 1.3.6.1.4.1.1466.115.121.1.26
SINGLE-VALUE )
olcObjectClasses: ( 1.2.40.0.13.1.15.110.4.4 NAME 'dcmArchiveDevice'
DESC 'DICOM Archive Device related information'
SUP top AUXILIARY
Expand Down Expand Up @@ -2862,6 +2871,8 @@ olcObjectClasses: ( 1.2.40.0.13.1.15.110.4.12 NAME 'dcmStorage'
dcmStorageThresholdExceeded $
dcmStorageThresholdExceedsPermanently $
dcmDeleterThreshold $
dcmDeleterThresholdMaxUsableSpace $
dcmDeleterThresholdBlocksFilePath $
dcmDeleteStudiesOlderThan $
dcmDeleteStudiesReceivedBefore $
dcmDeleteStudiesNotUsedSince $
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -562,6 +562,7 @@ private void writeStorageDescriptor(JsonWriter writer, Collection<StorageDescrip
writer.writeNotNullOrDef("dcmStorageClusterID", st.getStorageClusterID(), null);
writer.writeNotNullOrDef("dcmStorageThreshold", st.getStorageThreshold(), null);
writer.writeNotEmpty("dcmDeleterThreshold", st.getDeleterThresholdsAsStrings());
writer.writeNotEmpty("dcmDeleterThresholdMaxUsableSpace", st.getDeleterThresholdsMaxUsableSpaceAsStrings());
writer.writeNotEmpty("dcmProperty", st.getProperties());
writer.writeNotEmpty("dcmExternalRetrieveAET", st.getExternalRetrieveAETitles());
writer.writeNotNullOrDef("dcmExternalRetrieveInstanceAvailability",
Expand Down Expand Up @@ -2444,6 +2445,9 @@ private void loadStorageDescriptorFrom(ArchiveDeviceExtension arcDev, JsonReader
case "dcmCheckMountFilePath":
st.setCheckMountFilePath(reader.stringValue());
break;
case "dcmDeleterThresholdBlocksFilePath":
st.setDeleterThresholdBlocksFilePath(reader.stringValue());
break;
case "dcmFileOpenOption":
st.setFileOpenOptions(toOpenOptions(reader.stringArray()));
break;
Expand Down Expand Up @@ -2486,6 +2490,9 @@ private void loadStorageDescriptorFrom(ArchiveDeviceExtension arcDev, JsonReader
case "dcmDeleterThreshold":
st.setDeleterThresholdsFromStrings(reader.stringArray());
break;
case "dcmDeleterThresholdMaxUseableSpace":
st.setDeleterThresholdsMaxUseableSpaceFromStrings(reader.stringArray());
break;
case "dcmProperty":
st.setProperties(reader.stringArray());
break;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2816,6 +2816,8 @@ private Attributes storeTo(ConfigurationChanges.ModifiedObject ldapObj, StorageD
descriptor.isAltCreateDirectories(), false);
LdapUtils.storeNotNullOrDef(ldapObj, attrs, "dcmCheckMountFilePath",
descriptor.getCheckMountFilePath(), null);
LdapUtils.storeNotNullOrDef(ldapObj, attrs, "dcmDeleterThresholdBlocksFilePath",
descriptor.getDeleterThresholdBlocksFilePath(), null);
LdapUtils.storeNotEmpty(ldapObj, attrs, "dcmFileOpenOption",
descriptor.getFileOpenOptions(), StandardOpenOption.CREATE_NEW);
LdapUtils.storeNotNullOrDef(ldapObj, attrs, "dcmLocationStatus",
Expand Down Expand Up @@ -2844,6 +2846,8 @@ private Attributes storeTo(ConfigurationChanges.ModifiedObject ldapObj, StorageD
descriptor.getStorageThreshold(), null);
LdapUtils.storeNotEmpty(ldapObj, attrs, "dcmDeleterThreshold",
descriptor.getDeleterThresholdsAsStrings());
LdapUtils.storeNotEmpty(ldapObj, attrs, "dcmDeleterThresholdMaxUsableSpace",
descriptor.getDeleterThresholdsMaxUsableSpaceAsStrings());
LdapUtils.storeNotEmpty(ldapObj, attrs, "dcmProperty",
descriptor.getProperties());
LdapUtils.storeNotEmpty(ldapObj, attrs, "dcmExternalRetrieveAET",
Expand Down Expand Up @@ -2892,6 +2896,7 @@ private void loadStorageDescriptors(ArchiveDeviceExtension arcdev, String device
desc.setRetryCreateDirectories(LdapUtils.intValue(attrs.get("dcmRetryCreateDirectories"), 0));
desc.setAltCreateDirectories(LdapUtils.booleanValue(attrs.get("dcmAltCreateDirectories"), false));
desc.setCheckMountFilePath(LdapUtils.stringValue(attrs.get("dcmCheckMountFilePath"), null));
desc.setDeleterThresholdBlocksFilePath(LdapUtils.stringValue(attrs.get("dcmDeleterThresholdBlocksFilePath"), null));
desc.setFileOpenOptions(toOpenOptions(attrs.get("dcmFileOpenOption"), StandardOpenOption.CREATE_NEW));
desc.setLocationStatus(
LdapUtils.enumValue(LocationStatus.class, attrs.get("dcmLocationStatus"), LocationStatus.OK));
Expand All @@ -2912,6 +2917,8 @@ private void loadStorageDescriptors(ArchiveDeviceExtension arcdev, String device
desc.setStorageClusterID(LdapUtils.stringValue(attrs.get("dcmStorageClusterID"), null));
desc.setStorageThreshold(toStorageThreshold(attrs.get("dcmStorageThreshold")));
desc.setDeleterThresholdsFromStrings(LdapUtils.stringArray(attrs.get("dcmDeleterThreshold")));
desc.setDeleterThresholdsMaxUseableSpaceFromStrings(
LdapUtils.stringArray(attrs.get("dcmDeleterThresholdMaxUsableSpace")));
desc.setProperties(LdapUtils.stringArray(attrs.get("dcmProperty")));
desc.setExternalRetrieveAETitles(LdapUtils.stringArray(attrs.get("dcmExternalRetrieveAET")));
desc.setExternalRetrieveInstanceAvailability(LdapUtils.enumValue(
Expand Down Expand Up @@ -3006,6 +3013,8 @@ private List<ModificationItem> storeDiffs(ConfigurationChanges.ModifiedObject ld
prev.isAltCreateDirectories(), desc.isAltCreateDirectories(), false);
LdapUtils.storeDiffObject(ldapObj, mods, "dcmCheckMountFilePath",
prev.getCheckMountFilePath(), desc.getCheckMountFilePath(), null);
LdapUtils.storeDiffObject(ldapObj, mods, "dcmDeleterThresholdBlocksFilePath",
prev.getDeleterThresholdBlocksFilePath(), desc.getDeleterThresholdBlocksFilePath(), null);
LdapUtils.storeDiff(ldapObj, mods, "dcmFileOpenOption",
prev.getFileOpenOptions(), desc.getFileOpenOptions(), StandardOpenOption.CREATE_NEW);
LdapUtils.storeDiffObject(ldapObj, mods, "dcmLocationStatus",
Expand All @@ -3031,6 +3040,9 @@ private List<ModificationItem> storeDiffs(ConfigurationChanges.ModifiedObject ld
prev.getStorageThreshold(), desc.getStorageThreshold(), null);
LdapUtils.storeDiff(ldapObj, mods, "dcmDeleterThreshold",
prev.getDeleterThresholdsAsStrings(), desc.getDeleterThresholdsAsStrings());
LdapUtils.storeDiff(ldapObj, mods, "dcmDeleterThresholdMaxUseableSpace",
prev.getDeleterThresholdsMaxUsableSpaceAsStrings(),
desc.getDeleterThresholdsMaxUsableSpaceAsStrings());
LdapUtils.storeDiffProperties(ldapObj, mods, "dcmProperty", prev.getProperties(), desc.getProperties());
LdapUtils.storeDiff(ldapObj, mods, "dcmExternalRetrieveAET",
prev.getExternalRetrieveAETitles(), desc.getExternalRetrieveAETitles());
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -42,9 +42,10 @@
import org.dcm4che3.util.AttributesFormat;
import org.dcm4che3.util.StringUtils;

import java.io.IOException;
import java.io.InputStream;
import java.net.URI;
import java.nio.file.OpenOption;
import java.nio.file.StandardOpenOption;
import java.nio.file.*;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.time.Period;
Expand All @@ -70,6 +71,7 @@ public enum OnStoragePathAlreadyExists {
private AttributesFormat storagePathFormat = DEFAULT_ATTRIBUTES_FORMAT;
private OnStoragePathAlreadyExists onStoragePathAlreadyExists = OnStoragePathAlreadyExists.RANDOM_PATH;
private String checkMountFilePath;
private String deleterThresholdBlocksFilePath;
private OpenOption[] fileOpenOptions = { StandardOpenOption.CREATE_NEW };
private boolean altCreateDirectories;
private int retryCreateDirectories;
Expand Down Expand Up @@ -97,6 +99,7 @@ public enum OnStoragePathAlreadyExists {
private StorageDuration storageDuration = StorageDuration.PERMANENT;
private StorageThreshold storageThreshold;
private final ArrayList<DeleterThreshold> deleterThresholds = new ArrayList<>();
private final ArrayList<DeleterThreshold> deleterThresholdsMaxUseableSpace = new ArrayList<>();
private final Map<String, String> properties = new HashMap<>();
private final EnumMap<RetentionPeriod.DeleteStudies,List<RetentionPeriod>> retentionPeriods =
new EnumMap(RetentionPeriod.DeleteStudies.class);
Expand Down Expand Up @@ -153,6 +156,36 @@ public void setCheckMountFilePath(String checkMountFilePath) {
this.checkMountFilePath = checkMountFilePath;
}

public String getDeleterThresholdBlocksFilePath() {
return deleterThresholdBlocksFilePath;
}

public void setDeleterThresholdBlocksFilePath(String deleterThresholdBlocksFilePath) {
this.deleterThresholdBlocksFilePath = deleterThresholdBlocksFilePath;
}

public long parseDeleterThresholdBlocksFile() throws IOException {
if (deleterThresholdBlocksFilePath == null)
return 0L;

Path path = Paths.get(deleterThresholdBlocksFilePath);
long l = Files.size(path);
if (l == 0 && l >= 16)
return 0L;

byte[] bytes = new byte[(int) l];
try (InputStream is = Files.newInputStream(path)) {
is.read(bytes);
}
long blocks = 0L;
for (byte b : bytes) {
int digit = (b & 0xFF) - '0';
if (digit < 0 || digit > 9) break;
blocks = blocks * 10 + digit;
}
return blocks;
}

public OpenOption[] getFileOpenOptions() {
return fileOpenOptions;
}
Expand Down Expand Up @@ -377,29 +410,61 @@ public boolean hasDeleterThresholds() {
}

public String[] getDeleterThresholdsAsStrings() {
String[] ss = new String[deleterThresholds.size()];
for (int i = 0; i < ss.length; i++) {
ss[i] = deleterThresholds.get(i).toString();
}
return ss;
return getDeleterThresholdsAsStrings(deleterThresholds);
}

public List<DeleterThreshold> getDeleterThresholds() {
return deleterThresholds;
}

public void setDeleterThresholdsFromStrings(String... ss) {
setDeleterThresholdsFromStrings(ss, deleterThresholds);
}

public long getDeleterThresholdMinUsableSpace(Calendar cal) {
return getDeleterThresholdDiskSpace(cal, this.deleterThresholds);
}

public boolean hasDeleterThresholdMaxUsableSpace() {
return !deleterThresholdsMaxUseableSpace.isEmpty();
}

public String[] getDeleterThresholdsMaxUsableSpaceAsStrings() {
return getDeleterThresholdsAsStrings(deleterThresholdsMaxUseableSpace);
}

public List<DeleterThreshold> getDeleterThresholdsMaxUseableSpace() {
return deleterThresholdsMaxUseableSpace;
}

public void setDeleterThresholdsMaxUseableSpaceFromStrings(String... ss) {
setDeleterThresholdsFromStrings(ss, deleterThresholdsMaxUseableSpace);
}

public long getDeleterThresholdMaxUsableSpace(Calendar cal) {
return getDeleterThresholdDiskSpace(cal, this.deleterThresholdsMaxUseableSpace);
}

private static String[] getDeleterThresholdsAsStrings(ArrayList<DeleterThreshold> deleterThresholds) {
String[] ss = new String[deleterThresholds.size()];
for (int i = 0; i < ss.length; i++) {
ss[i] = deleterThresholds.get(i).toString();
}
return ss;
}

private static void setDeleterThresholdsFromStrings(String[] ss, ArrayList<DeleterThreshold> deleterThresholds) {
deleterThresholds.clear();
for (String s : ss) {
deleterThresholds.add(DeleterThreshold.valueOf(s));
}
Collections.sort(deleterThresholds);
}

public long getDeleterThresholdMinUsableSpace(Calendar cal) {
private static long getDeleterThresholdDiskSpace(Calendar cal, ArrayList<DeleterThreshold> deleterThresholds) {
for (DeleterThreshold deleterThreshold : deleterThresholds) {
if (deleterThreshold.match(cal))
return deleterThreshold.getMinUsableDiskSpace();
return deleterThreshold.getDiskSpace();
}
return -1L;
}
Expand Down
Loading

0 comments on commit 5b72b6d

Please sign in to comment.