draft-ietf-httpbis-rand-access-live.xml

<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE rfc [
<!ENTITY RFC2119 SYSTEM "http://www.rfc-editor.org/refs/bibxml/reference.RFC.2119.xml">
<!ENTITY RFC5234 SYSTEM "http://www.rfc-editor.org/refs/bibxml/reference.RFC.5234.xml">
<!ENTITY RFC7230 SYSTEM "http://www.rfc-editor.org/refs/bibxml/reference.RFC.7230.xml">
<!ENTITY RFC7233 SYSTEM "http://www.rfc-editor.org/refs/bibxml/reference.RFC.7233.xml">
<!ENTITY RFC7234 SYSTEM "http://www.rfc-editor.org/refs/bibxml/reference.RFC.7234.xml">
<!ENTITY RFC8216 SYSTEM "http://www.rfc-editor.org/refs/bibxml/reference.RFC.8216.xml">

<!-- Fudge for XMLmind which doesn't have this built in -->
<!ENTITY nbsp    "&#160;">
]>

<!-- Extra statement used by XSLT processors to control the output style. -->
<?xml-stylesheet type='text/xsl' href='rfc2629.xslt' ?>

<!-- Processing Instructions can be placed here but if you are editing
     with XMLmind (and maybe other XML editors) they are better placed
     after the rfc element start tag as shown below. -->

<rfc
    category="exp"
    ipr="trust200902"
    docName="draft-ietf-httpbis-rand-access-live-latest"
    xmlns:x="http://purl.org/net/xml2rfc/ext">

    <!-- Processing Instructions- PIs (for a complete list and description,
          see file http://xml.resource.org/authoring/README.html and below... -->

    <!-- Some of the more generally applicable PIs that most I-Ds might want to use -->

    <!-- Try to enforce the ID-nits conventions and DTD validity -->
    <?rfc strict="yes" ?>

    <!-- Items used when reviewing the document -->
    <?rfc comments="no" ?>  <!-- Controls display of <cref> elements -->
    <?rfc inline="no" ?>    <!-- When no, put comments at end in comments section,
                                 otherwise, put inline -->
    <?rfc editing="no" ?>   <!-- When yes, insert editing marks: editing marks consist of a
                                 string such as <29> printed in the blank line at the
                                 beginning of each paragraph of text. -->

    <!-- Create Table of Contents (ToC) and set some options for it.
         Note the ToC may be omitted for very short documents,but idnits insists on a ToC
         if the document has more than 15 pages. -->
   <?rfc toc="yes"?>
   <?rfc tocompact="yes"?> <!-- If "yes" eliminates blank lines before main section entries. -->
   <?rfc tocdepth="4"?>    <!-- Sets the number of levels of sections/subsections... in ToC -->

    <!-- Choose the options for the references.
         Some like symbolic tags in the references (and citations) and others prefer
         numbers. The RFC Editor always uses symbolic tags.
         The tags used are the anchor attributes of the references. -->
    <?rfc symrefs="yes"?>
    <?rfc sortrefs="yes" ?> <!-- If "yes", causes the references to be sorted in order of tags.
                                 This doesn't have any effect unless symrefs is "yes" also. -->

    <!-- These two save paper: Just setting compact to "yes" makes savings by not starting each
         main section on a new page but does not omit the blank lines between list items.
         If subcompact is also "yes" the blank lines between list items are also omitted. -->
    <?rfc compact="yes" ?>
    <?rfc subcompact="no" ?>
    <!-- end of list of popular I-D processing instructions -->

    <!-- ***** FRONT MATTER ***** -->
<front>
    <title>HTTP Random Access and Live Content</title>

    <author
        fullname="Craig Pratt"
        initials="C."
        surname="Pratt">
        <address>
            <postal>
                <city>Portland</city>
                <region>OR</region>
                <code>97229</code>
                <country>US</country>
            </postal>
        <email>pratt@acm.org</email>
        </address>
    </author>

    <author fullname="Darshak Thakore"
            initials="D."
            surname="Thakore" >
        <organization abbrev="CableLabs">CableLabs</organization>
        <address>
            <postal>
                <street>858 Coal Creek Circle</street>
                <city>Louisville</city>
                <region>CO</region>
                <code>80027</code>
                <country>US</country>
            </postal>
            <email>d.thakore@cablelabs.com</email>
        </address>
    </author>

    <author fullname="Barbara Stark"
            initials="B.H."
            surname="Stark" >
      <organization>AT&amp;T</organization>
      <address>
        <postal>
          <city>Atlanta</city>
          <region>GA</region>
          <country>US</country>
        </postal>
        <email>barbara.stark@att.com</email>
      </address>
    </author>

    <date year="2019" month="March"/>

    <!-- Meta-data Declarations -->

    <!-- Notice the use of &amp; as an escape for & which would otherwise
         start an entity declaration, whereas we want a literal &. -->
    <area>Applications and Real-Time</area>

    <!-- WG name at the upperleft corner of the doc,
         IETF fine for individual submissions.  You can also
         omit this element in which case in defaults to "Network Working Group" -
         a hangover from the ancient history of the IETF! -->
    <workgroup>HTTP</workgroup>

    <keyword>http range unit live aggregation</keyword>

    <!-- The DTD allows multiple area and workgroup elements but only the first one has any
         effect on output.  -->
    <!-- You can add <keyword/> elements here.  They will be incorporated into HTML output
         files in a meta tag but they have no effect on text or nroff output. -->


    <abstract>
        <t>
            To accommodate byte range requests for content that has
            data appended over time, this document defines semantics
            that allow a HTTP client and server to perform byte-range
      GET and HEAD requests that start at an arbitrary byte offset
      within the representation and ends at an indeterminate offset.
        </t>
    </abstract>

    <note title="Editorial Note (To be removed by RFC Editor before publication)">
      <t>
        Discussion of this draft takes place on the HTTPBIS working group mailing list
        (ietf-http-wg@w3.org), which is archived at <eref
        target="https://lists.w3.org/Archives/Public/ietf-http-wg/"/>.
      </t>
      <t>
        Working Group information can be found at <eref target="http://httpwg.github.io/"/>; source code and issues
        list for this draft can be found at
        <eref target="https://github.com/httpwg/http-extensions/labels/rand-access-live"/>.
      </t><!--
      <t>
        The changes in this draft are summarized in <xref target="change.log"/>.
      </t>-->
    </note>
</front>

<middle>

    <section anchor="introduction" title="Introduction">
        <t>
      Some Hypertext Transfer Protocol (HTTP) clients use byte-range requests (Range requests using the "bytes" Range Unit) to transfer select portions of large representations (<xref target="RFC7233"/>). And in some cases large representations require content to be continuously or periodically appended - such as representations consisting of live audio or video sources, blockchain databases, and log files. Clients cannot access the appended/live content using a Range request with the bytes range unit using the currently defined byte-range semantics without accepting performance or behavior sacrifices which are not acceptable for many applications.
        </t>
        <t>
			For instance, HTTP clients have the ability to access appended content on an indeterminate-length resource by transferring the entire representation from the beginning and continuing to read the appended content as it's made available. Obviously, this is highly inefficient for cases where the representation is large and only the most recently appended content is needed by the client.
		</t>
    	<t>
			Alternatively, clients can also access appended content by sending periodic open-ended bytes Range requests using the last-known end byte position as the range start. Performing low-frequency periodic bytes Range requests in this fashion (polling) introduces latency since the client will necessarily be somewhat behind the aggregated content - mimicking the behavior (and latency) of segmented content representations such as "HTTP Live Streaming" (HLS, <xref target="RFC8216"/>) or "Dynamic Adaptive Streaming over HTTP" (MPEG-DASH, <xref target="DASH"/>). And while performing these Range requests at higher frequency can reduce this latency, it also incurs more processing overhead and HTTP exchanges as many of the requests will return no content - since content is usually aggregated in groups of bytes (e.g. a video frame, audio sample, block, or log entry).
        </t>
        <t>
            This document describes a usage model for range requests which enables
            efficient retrieval of representations that are appended to over time
            by using large values and associated semantics for communicating
            range end positions. This model allows representations to be
            progressively delivered by servers as new content is added. It also
            ensures compatibility with servers and intermediaries that don't
            support this technique.
        </t>

        <section title="Requirements Language">
            <t>The key words &quot;MUST&quot;, &quot;MUST NOT&quot;,
            &quot;REQUIRED&quot;, &quot;SHALL&quot;, &quot;SHALL NOT&quot;,
            &quot;SHOULD&quot;, &quot;SHOULD NOT&quot;, &quot;RECOMMENDED&quot;,
            &quot;MAY&quot;, and &quot;OPTIONAL&quot; in this document are to be
            interpreted as described in <xref target="RFC2119">RFC 2119</xref>.
            </t>
        </section>

        <section title="Notational Conventions">
            <t>This document cites productions in Augmented Backus-Naur Form (ABNF) productions
            from <xref target="RFC7233"/>, using the notation defined in <xref target="RFC5234"/>.
            </t>
        </section>
    </section>

    <section anchor="definition" title="Performing Range requests on Random-Access Aggregating (&quot;live&quot;) Content">
        <t>
            This document recommends a two-step process for accessing resources
            that have indeterminate length representations.
        </t><t>
            Two steps are necessary because of limitations with the Range request
            header fields and the Content-Range response header fields. A server cannot
            know from a range request that a client wishes to receive a response
            that does not have a definite end. More critically, the header fields
            do not allow the server to signal that a resource has indeterminate
            length without also providing a fixed portion of the resource.
        </t><t>
            A client first learns that the resource has a representation of
            indeterminate length by requesting a range of the resource. The server
            responds with the range that is available, but indicates that the
            length of the representation is unknown using the existing
            Content-Range syntax. See <xref target="establishing-range" />
            for details and examples.
        </t><t>
            Once the client knows the resource has indeterminate length, it can
            request a range with a very large end position from the resource. The
            client chooses an explicit end value larger than can be transferred in
            the foreseeable term. A server which supports range requests of
            indeterminate length signals its understanding of the client's
            indeterminate range request by indicating that the range it is
            providing has a range end that exactly matches the client's requested
            range end rather than a range that is bounded by what is currently
            available. See <xref target="live-range-requests" /> for details.
    </t>

      <section anchor="establishing-range" title="Establishing the Randomly Accessible Byte Range">
      <t>
        Establishing if a representation is continuously aggregating ("live") and determining the randomly-accessible byte range can both be determined using the existing definition for an open-ended byte-range request. Specifically, <xref x:sec="2.1" x:fmt="of" target="RFC7233"/> defines a byte-range request of the form:
      </t>
<figure><artwork type="abnf">
   byte-range-spec = first-byte-pos "-" [ last-byte-pos ]
</artwork></figure>
      <t>
      which allows a client to send a HEAD request with a first-byte-pos and leave last-byte-pos absent. A
      server that receives a satisfiable byte-range request (with first-byte-pos smaller than the current
      representation length) may respond with a 206 status code (Partial Content) with a Content-Range
      header field indicating the currently satisfiable byte range. For example:
      </t>
<figure><artwork type="message/http; msgtype=&#34;request&#34;" x:indent-with="    ">
HEAD /resource HTTP/1.1
Host: example.com
Range: bytes=0-

</artwork></figure>
      <t>
      returns a response of the form:
      </t>
<figure><artwork type="message/http; msgtype=&#34;response&#34;" x:indent-with="    ">
HTTP/1.1 206 Partial Content
Content-Range: bytes 0-1234567/*
</artwork></figure>
      <t>
      from the server indicating that (1) the complete representation length is unknown (via the "*" in place of the complete-length field) and (2) that only bytes 0-1234567 were accessible at the time the request was processed by the server. The client can infer from this response that bytes 0-1234567 of the representation can be requested and returned in a timely fashion (the bytes are immediately available).
      </t>
    </section>
    <section anchor="live-range-requests" title="Byte-Range Requests Beyond the Randomly Accessible Byte Range">
      <t>
        Once a client has determined that a representation has an indeterminate length and established the byte range that can be accessed, it may want to perform a request with a start position within the randomly-accessible content range and an end position at an indefinite "live" point - a point where the byte-range GET request is fulfilled on-demand as the content is aggregated.
      </t>
      <t>
        For example, for a large video asset, a client may wish to start a content transfer from the video "key" frame immediately before the point of aggregation and continue the content transfer indefinitely as content is aggregated - in order to support low-latency startup of a live video stream.
      </t>
      <t>
        Unlike a byte-range Range request, a byte-range Content-Range response header field cannot be "open ended", per <xref x:sec="4.2" x:fmt="of" target="RFC7233"/>:
      </t>
<figure><artwork type="abnf">
   byte-content-range  = bytes-unit SP
                        ( byte-range-resp / unsatisfied-range )

   byte-range-resp     = byte-range "/" ( complete-length / "*" )
   byte-range          = first-byte-pos "-" last-byte-pos
   unsatisfied-range   = "*/" complete-length

   complete-length     = 1*DIGIT
</artwork></figure>
      <t>
        Specifically, last-byte-pos is required in byte-range. So in order to preserve interoperability with existing HTTP clients, servers, proxies, and caches, this document proposes a mechanism for a client to indicate support for handling an indeterminate-length byte-range response, and a mechanism for a server to indicate if/when it's providing an  indeterminate-length response.
      </t>
      <t>
        A client can indicate support for handling indeterminate-length byte-range responses by providing a very large value for the last-byte-pos in the byte-range request. For example, a client can perform a byte-range GET request of the form:
      </t>
<figure><artwork type="message/http; msgtype=&#34;request&#34;" x:indent-with="    ">
GET /resource HTTP/1.1
Host: example.com
Range: bytes=1230000-999999999999

</artwork></figure>
      <t>
        where the last-byte-pos in the Request is much larger than the last-byte-pos returned in response to an open-ended byte-range HEAD request, as described above, and much larger than the expected maximum size of the representation. See <xref target="Security" /> for range value considerations.
      </t>
      <t>
        In response, a server may indicate that it is supplying a continuously aggregating ("live") response by supplying the client request's last-byte-pos in the Content-Range response header field.
      </t>
      <t>
        For example:
      </t>
<figure><artwork type="message/http; msgtype=&#34;request&#34;" x:indent-with="    ">
GET /resource HTTP/1.1
Host: example.com
Range: bytes=1230000-999999999999

</artwork></figure>
      <t>
        returns
      </t>
<figure><artwork type="message/http; msgtype=&#34;response&#34;" x:indent-with="    ">
HTTP/1.1 206 Partial Content
Content-Range: bytes 1230000-999999999999/*
</artwork></figure>
      <t>
        from the server to indicate that the response will start at byte 1230000 and continues indefinitely to include all aggregated content, as it becomes available.
      </t>
      <t>
        A server that doesn't support or supply a continuously aggregating ("live") response will supply the currently satisfiable byte range, as it would with an open-ended byte request.
      </t>
      <t>
        For example:
      </t>
<figure><artwork type="message/http; msgtype=&#34;request&#34;" x:indent-with="    ">
GET /resource HTTP/1.1
Host: example.com
Range: bytes=1230000-999999999999

</artwork></figure>
      <t>
        will return
      </t>
<figure><artwork type="message/http; msgtype=&#34;response&#34;" x:indent-with="    ">
HTTP/1.1 206 Partial Content
Content-Range: bytes 1230000-1234567/*
</artwork></figure>
      <t>
        from the server to indicate that the response will start at byte 1230000 and end at byte 1234567 and will not include any aggregated content. This is the response expected from a typical HTTP server - one that doesn't support byte-range requests on aggregating content.
      </t>
      <t>
        A client that doesn't receive a response indicating it is continuously aggregating must use other means to access aggregated content (e.g. periodic byte-range polling).
          </t>
      <t>
        A server that does return a continuously aggregating ("live") response should return data using chunked transfer coding and not provide a Content-Length header field. A 0-length chunk indicates the end of the transfer, per <xref x:sec="4.1" x:fmt="of" target="RFC7230"/>.
          </t>
    </section>
  </section>
    <section anchor="other-applications" title="Other Applications of Random-Access Aggregating Content">
      <section anchor="starting-at-live" title="Requests Starting at the Aggregation (&quot;Live&quot;) Point">
      <t>
          A client that wishes to only receive newly-aggregated portions of a
          resource (i.e., start at the "live" point), can use a HEAD request to
          learn what range the server has currently available and initiate an
          indeterminate-length transfer. For example:
      </t>
<figure><artwork type="message/http; msgtype=&#34;request&#34;" x:indent-with="    ">
HEAD /resource HTTP/1.1
Host: example.com
Range: bytes=0-

</artwork></figure>
        <t>
            With the Content-Range response header field indicating the range (or ranges) available. For example:
        </t>
<figure><artwork type="message/http; msgtype=&#34;response&#34;" x:indent-with="    ">
206 Partial Content
Content-Range: bytes 0-1234567/*
</artwork></figure>
        <t>
            The client can then issue a request for a range starting at the end
            value (using a very large value for the end of a range) and receive
            only new content.
        </t>
<figure><artwork type="message/http; msgtype=&#34;request&#34;" x:indent-with="    ">
GET /resource HTTP/1.1
Host: example.com
Range: bytes=1234567-999999999999

</artwork></figure>
        <t>
          with a server returning a Content-Range response indicating that an indeterminate-length response body will be provided
        </t>
<figure><artwork type="message/http; msgtype=&#34;response&#34;" x:indent-with="    ">
206 Partial Content
Content-Range: bytes 1234567-999999999999/*
</artwork></figure>
      </section>
      <section anchor="shift-buffers" title="Shift Buffer Representations">
      <t>
        Some representations lend themselves to front-end content removal in addition to aggregation. While still supporting random access, representations of this type have a portion at the beginning (the "0" end) of the randomly-accessible region that become inaccessible over time. Examples of this kind of representation would be an audio-video time-shift buffer or a rolling log file.
      </t>
      <t>
        For example a Range request containing:
      </t>
<figure><artwork type="message/http; msgtype=&#34;request&#34;" x:indent-with="    ">
HEAD /resource HTTP/1.1
Host: example.com
Range: bytes=0-

</artwork></figure>
      <t>
        returns
      </t>
<figure><artwork type="message/http; msgtype=&#34;response&#34;" x:indent-with="    ">
206 Partial Content
Content-Range: bytes 1000000-1234567/*
</artwork></figure>
      <t>
        indicating that the first 1000000 bytes were not accessible at the time the HEAD request was processed. Subsequent HEAD requests could return:
      </t>
<figure><artwork type="example" x:indent-with="    ">
Content-Range: bytes 1000000-1234567/*
</artwork></figure>
<figure><artwork type="example" x:indent-with="    ">
Content-Range: bytes 1010000-1244567/*
</artwork></figure>
<figure><artwork type="example" x:indent-with="    ">
Content-Range: bytes 1020000-1254567/*
</artwork></figure>
      <t>
        Note though that the difference between the first-byte-pos and last-byte-pos need not be constant.
      </t>
      <t>
        The client could then follow-up with a GET Range request containing
      </t>
<figure><artwork type="message/http; msgtype=&#34;request&#34;" x:indent-with="    ">
GET /resource HTTP/1.1
Host: example.com
Range: bytes=1020000-999999999999

</artwork></figure>
      <t>
        with the server returning
      </t>
<figure><artwork type="message/http; msgtype=&#34;response&#34;" x:indent-with="    ">
206 Partial Content
Content-Range: bytes 1020000-999999999999/*
</artwork></figure>
      <t>
        with the response body returning bytes 1020000-1254567 immediately and aggregated ("live") data being returned as the content is aggregated.
      </t>
      <t>
        A server that doesn't support or supply a continuously aggregating ("live") response will supply the currently satisfiable byte range, as it would with an open-ended byte request.
      </t>
      <t>
        For example:
      </t>
<figure><artwork type="message/http; msgtype=&#34;request&#34;" x:indent-with="    ">
GET /resource HTTP/1.1
Host: example.com
Range: bytes=0-999999999999

</artwork></figure>
      <t>
        will return
      </t>
<figure><artwork type="message/http; msgtype=&#34;response&#34;" x:indent-with="    ">
HTTP/1.1 206 Partial Content
Content-Range: bytes 1020000-1254567/*
</artwork></figure>
      <t>
        from the server to indicate that the response will start at byte 1020000, end at byte 1254567, and will not include any aggregated content. This is the response expected from a typical HTTP server - one that doesn't support byte-range requests on aggregating content.
      </t>
      <t>
          Note that responses to GET requests against shift-buffer representations using Range
          can be cached by intermediaries, since the Content-Range response header indicates
          which portion of the representation is being returned in the response body. However
          GET requests without a Range header cannot be cached since the first byte of the response
          body can vary from request to request. To ensure Range-less GET requests against
          shift-buffer representations are not cached, servers hosting a shift-buffer
          representation should either not return a 200-level response (e.g. sending a
          300-level redirect response with a URI that represents the current start of the
          shift-buffer) or indicate the response is non-cacheable. See HTTP Caching
          (<xref target="RFC7234"/>) for details on HTTP cache control.
      </t>
      </section>
    </section>

<!-- Possibly a 'Contributors' section ... -->

    <section anchor="RecommendedVLV" title="Recommendations for Very Large Values">
        <t>
            While it would be ideal to define a single numerical Very Large Value, there's no single value that would work for all applications and platforms. e.g. JavaScript numbers cannot represent all integer values above 2^^53, so a JavaScript application may want to use 2^^53-1 for a Very Large Value. This value, however, would not be sufficient for all applications, such as continuously-streaming high-bitrate streams. So the value 2^^53-1 (9007199254740991) is recommended as a Very Large Value unless an application has a good justification to use a smaller or larger value. e.g. If it's always known that the resource won't exceed a value smaller than the recommended Very Large Value for an application, a smaller value can be used. And if it's likely that an application will utilize resources larger than the recommended Very Large Value - such as a continuously aggregating high-bitrate media stream - a larger value should be used.
        </t>
        <t>
            Note that, in accordance with the semantics defined above, servers that support random-access live content will need to return the last-byte-pos provided in the Range request in some cases - even if the last-byte-pos cannot be represented as a numerical value internally by the server. As is the case with any live/continuously aggregating resource, the server should terminate the content transfer when the end of the resource is reached - whether the end is due to termination of the content source or the content length exceeds the server's maximum representation length.
        </t>
    </section>

    <section anchor="IANA" title="IANA Considerations">
      <t>This document has no actions for IANA.</t>
    </section>

    <section anchor="Security" title="Security Considerations">
        <t>
            As described above, servers need to be prepared to receive last-byte-pos values in Range requests that are numerically larger than the server implementation supports - and return these values in Content-Range response header fields. Servers should check the last-byte-pos value before converting and storing them into numeric form to ensure the value doesn't cause an overflow or index incorrect data. The simplest way to satisfy the live-range semantics defined in this document without potential overflow issues is to store the last-byte-pos as a string value and return it in the byte-range Content-Range response header's last-byte-pos field.
        </t>
    </section>
</middle>

<!--  *****BACK MATTER ***** -->
<back>
    <!-- References split to informative and normative -->
    <references title="Normative References">
        &RFC2119;
        &RFC7230;
        &RFC7233;
        &RFC7234;
    </references>

    <references title="Informative References">
        &RFC5234;
        &RFC8216;
        <reference anchor="DASH" target="http://standards.iso.org/ittf/PubliclyAvailableStandards/c065274_ISO_IEC_23009-1_2014.zip">
          <front>
            <title>Information technology -- Dynamic adaptive streaming over HTTP (DASH) -- Part 1: Media presentation description and segment formats
            </title>
            <author><organization>ISO</organization></author>
            <date month="May" year="2014" />
          </front>
          <seriesInfo name="ISO/IEC" value="23009-1:2014" />
        </reference>
    </references>

    <section anchor="Acknowledgements" title="Acknowledgements" numbered="false">
        <t>
      Mark Nottingham, Patrick McManus, Julian Reschke, Remy Lebeau, Rodger Combs, Thorsten Lohmar, Martin Thompson, Adrien de Croy, K. Morgan, Roy T. Fielding, Jeremy Poulter.
        </t>
    </section>

</back>

</rfc>