st2101: initial implementation

README.md

@@ -67,6 +67,7 @@ The table below lists the status of currently-supported standards:
 | ST 1907    | Motion Imagery Metadata (MIMD): Payload                                         | Implemented as of ST 1907.1. No interoperability testing.                                                          |                                                                                                                          |
 | ST 1908    | Motion Imagery Metadata (MIMD): Imager System                                   | Implemented as of ST 1908.1. No interoperability testing.                                                          |                                                                                                                          |
 | ST 1909    | Metadata Overlay for Visualization                                              | Mostly implemented as of ST 1909.1. No support for next zoom or the reticle.                                       | [#97](https://github.com/WestRidgeSystems/jmisb/issues/97)                                                               |
+| ST 2101    | Core Identifier for Class 1/Class 2 Motion Imagery                              | Implemented as of ST 2101. Support is FFmpeg codec-specific.                                                       |                                                                                                                          |
 
 jmisb aims to be cross-platform to run on any modern operating system. However,
 since efficient video coding tends to leverage natively-compiled binaries, currently

api-ffmpeg/src/main/java/org/jmisb/api/video/CodecConfigurations.java

@@ -16,6 +16,7 @@ private CodecConfigurations() {
         codecs.add(
                 new CodecConfiguration("hevc_nvenc", "NVIDIA", CodecIdentifier.H265)
                         .addCodecOption("tune", "zerolatency")
+                        .addCodecOption("udu_sei", "1")
                         .addCodecOption("preset", "ll"));
         codecs.add(
                 new CodecConfiguration("libx265", "x265", CodecIdentifier.H265)
@@ -24,6 +25,7 @@ private CodecConfigurations() {
         codecs.add(
                 new CodecConfiguration("h264_nvenc", "NVIDIA", CodecIdentifier.H264)
                         .addCodecOption("tune", "zerolatency")
+                        .addCodecOption("udu_sei", "1")
                         .addCodecOption("preset", "fast"));
         codecs.add(
                 new CodecConfiguration("h264_qsv", "Intel QuickSync", CodecIdentifier.H264)
@@ -38,6 +40,7 @@ private CodecConfigurations() {
         codecs.add(
                 new CodecConfiguration("libx264", "x264", CodecIdentifier.H264)
                         .addCodecOption("tune", "zerolatency")
+                        .addCodecOption("udu_sei", "1")
                         .addCodecOption("preset", "ultrafast"));
     }
 

api-ffmpeg/src/main/java/org/jmisb/api/video/VideoDecodeThread.java

@@ -39,6 +39,8 @@
 import org.jmisb.api.klv.st0603.ST0603TimeStamp;
 import org.jmisb.api.klv.st0603.TimeStatus;
 import org.jmisb.api.klv.st0604.TimeStampUtilities;
+import org.jmisb.api.klv.st1204.CoreIdentifier;
+import org.jmisb.api.klv.st2101.ST2101Converter;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -186,6 +188,7 @@ public void run() {
 
                         ST0603TimeStamp motionImageryTimeStamp = null;
                         TimeStatus timeStatus = null;
+                        CoreIdentifier miisCoreId = null;
                         AVFrameSideData sideData =
                                 av_frame_get_side_data(avFrame, AV_FRAME_DATA_SEI_UNREGISTERED);
                         if (sideData != null) {
@@ -196,7 +199,7 @@ public void run() {
                                 timeStatus = TimeStampUtilities.decodeTimeStatus(sideDataBytes);
                                 motionImageryTimeStamp =
                                         TimeStampUtilities.decodePrecisionTimeStamp(sideDataBytes);
-                                // TODO: check the ST2101 UUID case.
+                                miisCoreId = ST2101Converter.decodeCoreId(sideDataBytes);
                             }
                         }
 
@@ -206,6 +209,7 @@ public void run() {
                             VideoFrame videoFrame = new VideoFrame(image, pts);
                             videoFrame.setTimeStamp(motionImageryTimeStamp);
                             videoFrame.setTimeStatus(timeStatus);
+                            videoFrame.setMiisCoreId(miisCoreId);
                             queued = inputStream.queueVideoFrame(videoFrame, 20);
                         }
                     } else if (ret != -11 && ret != -35) // -11 = EAGAIN, -35 = EDEADLK

api-ffmpeg/src/main/java/org/jmisb/api/video/VideoFrame.java

@@ -3,13 +3,15 @@
 import java.awt.image.BufferedImage;
 import org.jmisb.api.klv.st0603.ST0603TimeStamp;
 import org.jmisb.api.klv.st0603.TimeStatus;
+import org.jmisb.api.klv.st1204.CoreIdentifier;
 
 /** An uncompressed video frame. */
 public class VideoFrame {
     private final BufferedImage bufferedImage;
     private final double pts;
     private ST0603TimeStamp timeStamp = null;
     private TimeStatus timeStatus = null;
+    private CoreIdentifier miisCoreId = null;
 
     /**
      * Create a video frame.
@@ -85,4 +87,27 @@ public TimeStatus getTimeStatus() {
     public void setTimeStatus(TimeStatus timeStatus) {
         this.timeStatus = timeStatus;
     }
+
+    /**
+     * Get the MIIS Core Identifier.
+     *
+     * <p>Not all frames will have a MIIS Core Identifier. ST 2101 requires it at least every 30
+     * seconds, or when it changes. Also, this concept is fairly new, and there are a lot of older
+     * imagery sources.
+     *
+     * @return the core identifier for this frame, or null if there is no core identifier in the
+     *     frame.
+     */
+    public CoreIdentifier getMiisCoreId() {
+        return miisCoreId;
+    }
+
+    /**
+     * Set the core identifier for this frame.
+     *
+     * @param miisCoreId the core identifier to set on the frame.
+     */
+    public void setMiisCoreId(CoreIdentifier miisCoreId) {
+        this.miisCoreId = miisCoreId;
+    }
 }

api-ffmpeg/src/main/java/org/jmisb/api/video/VideoInput.java

@@ -298,6 +298,9 @@ private static VideoFrame deepCopy(VideoFrame frame) {
         if (frame.getTimeStatus() != null) {
             videoFrame.setTimeStatus(frame.getTimeStatus().deepCopy());
         }
+        if (frame.getMiisCoreId() != null) {
+            videoFrame.setMiisCoreId(frame.getMiisCoreId());
+        }
         return videoFrame;
     }
 

api-ffmpeg/src/main/java/org/jmisb/api/video/VideoOutput.java

@@ -22,11 +22,13 @@
 import static org.bytedeco.ffmpeg.global.avformat.avio_close;
 import static org.bytedeco.ffmpeg.global.avutil.AVMEDIA_TYPE_DATA;
 import static org.bytedeco.ffmpeg.global.avutil.AVMEDIA_TYPE_VIDEO;
+import static org.bytedeco.ffmpeg.global.avutil.AV_FRAME_DATA_SEI_UNREGISTERED;
 import static org.bytedeco.ffmpeg.global.avutil.AV_PIX_FMT_BGR24;
 import static org.bytedeco.ffmpeg.global.avutil.AV_PIX_FMT_YUV420P;
 import static org.bytedeco.ffmpeg.global.avutil.av_d2q;
 import static org.bytedeco.ffmpeg.global.avutil.av_frame_alloc;
 import static org.bytedeco.ffmpeg.global.avutil.av_frame_free;
+import static org.bytedeco.ffmpeg.global.avutil.av_frame_new_side_data;
 import static org.bytedeco.ffmpeg.global.avutil.av_image_alloc;
 import static org.bytedeco.ffmpeg.global.avutil.av_image_fill_arrays;
 import static org.bytedeco.ffmpeg.global.avutil.av_inv_q;
@@ -51,11 +53,13 @@
 import org.bytedeco.ffmpeg.avformat.AVStream;
 import org.bytedeco.ffmpeg.avutil.AVDictionary;
 import org.bytedeco.ffmpeg.avutil.AVFrame;
+import org.bytedeco.ffmpeg.avutil.AVFrameSideData;
 import org.bytedeco.ffmpeg.avutil.AVRational;
 import org.bytedeco.ffmpeg.swscale.SwsContext;
 import org.bytedeco.javacpp.BytePointer;
 import org.bytedeco.javacpp.DoublePointer;
 import org.bytedeco.javacpp.PointerPointer;
+import org.jmisb.api.klv.st2101.ST2101Converter;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -359,12 +363,13 @@ void cleanup() {
     }
 
     /**
-     * Copy a BufferedImage to avFrameDst, transforming to the pixel format required by the codec.
+     * Copy a VideoFrame to avFrameDst, transforming to the pixel format required by the codec.
      *
-     * @param image The input image
+     * @param frame The input video frame
      * @throws IOException If the frame could not be written
      */
-    private void convert(BufferedImage image) throws IOException {
+    private void convert(VideoFrame frame) throws IOException {
+        BufferedImage image = frame.getImage();
         // If needed, convert to TYPE_3BYTE_BGR format (TODO: is there a more efficient way?)
         BufferedImage inputImage = image;
         if (image.getType() != BufferedImage.TYPE_3BYTE_BGR) {
@@ -450,6 +455,14 @@ private void convert(BufferedImage image) throws IOException {
                 inputImage.getHeight(),
                 new PointerPointer(avFrameDst),
                 avFrameDst.linesize());
+
+        if (frame.getMiisCoreId() != null) {
+            byte[] sideData = ST2101Converter.coreIdToSideDataBytes(frame.getMiisCoreId());
+            AVFrameSideData st2101_sei =
+                    av_frame_new_side_data(
+                            avFrameDst, AV_FRAME_DATA_SEI_UNREGISTERED, sideData.length);
+            st2101_sei.data().put(sideData, 0, sideData.length);
+        }
     }
 
     /**
@@ -499,7 +512,7 @@ AVPacket convert(MetadataFrame frame) {
      * @throws IOException if an error occurs
      */
     void encodeFrame(VideoFrame frame) throws IOException {
-        convert(frame.getImage());
+        convert(frame);
 
         // Convert PTS in seconds to PTS in "time base" units
         long pts = Math.round(frame.getPts() / av_q2d(videoStream.time_base()));

api-ffmpeg/src/test/java/org/jmisb/api/video/VideoFileInputOutputIT.java

@@ -7,8 +7,10 @@
 import java.util.List;
 import java.util.SortedMap;
 import java.util.TreeMap;
+import java.util.UUID;
 import javax.imageio.ImageIO;
 import org.jmisb.api.klv.IMisbMessage;
+import org.jmisb.api.klv.st1204.CoreIdentifier;
 import org.jmisb.core.video.TimingUtils;
 import org.jmisb.st0102.Classification;
 import org.jmisb.st0102.CountryCodingMethod;
@@ -291,7 +293,11 @@ private void createFile(
 
                 UasDatalinkMessage message = new UasDatalinkMessage(values);
 
-                output.addVideoFrame(new VideoFrame(image, pts));
+                VideoFrame frame = new VideoFrame(image, pts);
+                CoreIdentifier miisCoreId = new CoreIdentifier();
+                miisCoreId.setMinorUUID(UUID.randomUUID());
+                frame.setMiisCoreId(miisCoreId);
+                output.addVideoFrame(frame);
                 output.addMetadataFrame(new MetadataFrame(message, pts));
                 pts += frameDuration;
             }

api-ffmpeg/src/test/java/org/jmisb/api/video/VideoFrameTest.java

@@ -1,8 +1,10 @@
 package org.jmisb.api.video;
 
 import java.awt.image.BufferedImage;
+import java.util.UUID;
 import org.jmisb.api.klv.st0603.ST0603TimeStamp;
 import org.jmisb.api.klv.st0603.TimeStatus;
+import org.jmisb.api.klv.st1204.CoreIdentifier;
 import org.testng.Assert;
 import org.testng.annotations.Test;
 
@@ -16,16 +18,23 @@ public void testBasic() {
         ST0603TimeStamp timestamp = new ST0603TimeStamp(1547483647000000L);
         frame1.setTimeStamp(timestamp);
         VideoFrame frame2 = new VideoFrame(image2, 0.033);
+        CoreIdentifier miisCoreIdentifier = new CoreIdentifier();
+        miisCoreIdentifier.setMinorUUID(UUID.randomUUID());
         frame2.setTimeStatus(new TimeStatus());
+        frame2.setMiisCoreId(miisCoreIdentifier);
 
         Assert.assertEquals(frame1.getPts(), 0.0);
         Assert.assertEquals(frame2.getPts(), 0.033);
 
         Assert.assertEquals(frame1.getImage(), image1);
         Assert.assertEquals(frame1.getTimeStamp().getDisplayableValue(), "1547483647000000");
+        Assert.assertNull(frame1.getMiisCoreId());
         Assert.assertEquals(frame2.getImage(), image2);
         Assert.assertFalse(frame2.getTimeStatus().isDiscontinuity());
         Assert.assertFalse(frame2.getTimeStatus().isReverse());
         Assert.assertFalse(frame2.getTimeStatus().isLocked());
+        Assert.assertNotNull(frame2.getMiisCoreId());
+        Assert.assertEquals(
+                frame2.getMiisCoreId().getMinorUUID(), miisCoreIdentifier.getMinorUUID());
     }
 }

api/src/main/java/module-info.java

@@ -17,4 +17,5 @@
     exports org.jmisb.api.klv.st1201;
     exports org.jmisb.api.klv.st1204;
     exports org.jmisb.api.klv.st1303;
+    exports org.jmisb.api.klv.st2101;
 }

api/src/main/java/org/jmisb/api/klv/st0603/TimeStatus.java

@@ -164,4 +164,25 @@ public TimeStatus deepCopy() {
         copy.setDiscontinuity(this.isDiscontinuity(), this.isReverse());
         return copy;
     }
+
+    @Override
+    public String toString() {
+        StringBuilder sb = new StringBuilder();
+        if (isLocked()) {
+            sb.append("Locked | ");
+        } else {
+            sb.append("Not locked | ");
+        }
+        if (isDiscontinuity()) {
+            sb.append("Discontinuity - ");
+            if (isReverse()) {
+                sb.append("Reverse");
+            } else {
+                sb.append("Forward");
+            }
+        } else {
+            sb.append("No discontinuity");
+        }
+        return sb.toString();
+    }
 }

api/src/main/java/org/jmisb/api/klv/st1204/CoreIdentifier.java

@@ -391,6 +391,11 @@ public String getTextRepresentation() {
         return sb.toString();
     }
 
+    @Override
+    public String toString() {
+        return getTextRepresentation();
+    }
+
     private void parseVersionAndUsage(String versionAndUsage) {
         int versionAndUsageValue = Integer.parseInt(versionAndUsage, 16);
         setVersion(versionAndUsageValue >> 8);

api/src/main/java/org/jmisb/api/klv/st2101/ST2101Converter.java

@@ -0,0 +1,93 @@
+package org.jmisb.api.klv.st2101;
+
+import java.util.Arrays;
+import org.jmisb.api.klv.st1204.CoreIdentifier;
+
+/**
+ * Utility class for ST 2101 conversion of MIIS Core Identifier.
+ *
+ * <p>This class is not intended to be instantiated. Instead, use the required static methods
+ * directly.
+ */
+public class ST2101Converter {
+
+    private static final byte[] ST2101_UUID_BYTES =
+            new byte[] {
+                (byte) 0xa5,
+                (byte) 0x50,
+                (byte) 0x52,
+                (byte) 0xaf,
+                (byte) 0x52,
+                (byte) 0x16,
+                (byte) 0x5f,
+                (byte) 0x45,
+                (byte) 0xa3,
+                (byte) 0x18,
+                (byte) 0x1c,
+                (byte) 0xfc,
+                (byte) 0x7a,
+                (byte) 0xbb,
+                (byte) 0xc2,
+                (byte) 0x67
+            };
+
+    private static final int IDENTIFIER_OFFSET = 0;
+    private static final int IDENTIFIER_LENGTH = ST2101_UUID_BYTES.length;
+    private static final int MINIMUM_CORE_ID_LENGTH = 18;
+    private static final int MINIMUM_LENGTH = IDENTIFIER_LENGTH + MINIMUM_CORE_ID_LENGTH;
+
+    private ST2101Converter() {}
+
+    /**
+     * Convert a MIIS Core Identifier to ST 2101 SEI format.
+     *
+     * @param miisCoreId the Core Identifier to convert.
+     * @return encoded UUID and Core Identifier as a byte array.
+     */
+    public static byte[] coreIdToSideDataBytes(CoreIdentifier miisCoreId) {
+        byte[] st2101value = miisCoreId.getRawBytesRepresentation();
+        byte[] bytes = new byte[ST2101_UUID_BYTES.length + st2101value.length];
+        System.arraycopy(ST2101_UUID_BYTES, 0, bytes, 0, ST2101_UUID_BYTES.length);
+        System.arraycopy(st2101value, 0, bytes, ST2101_UUID_BYTES.length, st2101value.length);
+        return bytes;
+    }
+
+    /**
+     * Decode the MIIS Core Identifier from an encoded byte array.
+     *
+     * <p>To be valid, this will be the User Data Unregistered SEI entry, consisting of the UUID
+     * provided in ST 2101, followed by a MIIS Core Identifier.
+     *
+     * @param sideDataBytes the byte array of encoded data, beginning with the ST 2101 UUID.
+     * @return MIIS Core Identifier, or null if the byte array does not match the expected structure
+     *     or content.
+     */
+    public static CoreIdentifier decodeCoreId(byte[] sideDataBytes) {
+        if (isRequiredLength(sideDataBytes) && isCoreIdentifierSEI(sideDataBytes)) {
+            byte[] coreIdentifierBytes =
+                    Arrays.copyOfRange(sideDataBytes, IDENTIFIER_LENGTH, sideDataBytes.length);
+            return CoreIdentifier.fromBytes(coreIdentifierBytes);
+        } else {
+            return null;
+        }
+    }
+
+    private static boolean isRequiredLength(byte[] sideDataBytes) {
+        return sideDataBytes.length >= MINIMUM_LENGTH;
+    }
+
+    private static boolean isCoreIdentifierSEI(byte[] sideDataBytes) {
+        return sideDataBytesStartsWithIdentifier(sideDataBytes, ST2101_UUID_BYTES);
+    }
+
+    private static boolean sideDataBytesStartsWithIdentifier(
+            byte[] sideDataBytes, byte[] identifier) {
+        return Arrays.equals(
+                sideDataBytes,
+                IDENTIFIER_OFFSET,
+                IDENTIFIER_LENGTH,
+                identifier,
+                IDENTIFIER_OFFSET,
+                IDENTIFIER_LENGTH);
+    }
+}