Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

How to get response headers only, without the body? #262

Open
quantimnot opened this issue Mar 1, 2022 · 3 comments
Open

How to get response headers only, without the body? #262

quantimnot opened this issue Mar 1, 2022 · 3 comments

Comments

@quantimnot
Copy link

My Use Case

I needed to calculate the storage requirements for a very large set of archives hosted on a website.
The archives are retrieved with a POST method, so I can't use HEAD.

My Solution

Make the POST call and close the connection after the response headers are received.

I didn't see a means to accomplish this with std/httpclient or chronos/apps/http/httpclient. So this is what I did:

include chronos/apps/http/httpclient

proc getHeaderOnlyResponse(req: HttpClientRequestRef): Future[HttpTable] {.
     async.} =
  var buffer: array[HttpMaxHeadersSize, byte]
  let bytesRead =
    try:
      await req.connection.reader.readUntil(addr buffer[0],
                                            len(buffer), HeadersMark).wait(
                                            req.session.headersTimeout)
    except CancelledError as exc:
      raise exc
    except AsyncTimeoutError:
      raiseHttpReadError("Reading response headers timed out")
    except AsyncStreamError:
      raiseHttpReadError("Could not read response headers")

  ## Process response headers.
  let resp = parseResponse(buffer, false)
  if resp.failed():
    raiseHttpReadError("Invalid headers received")

  let headers =
    block:
      var res = HttpTable.init()
      for key, value in resp.headers(buffer):
        res.add(key, value)
      if res.count(ContentTypeHeader) > 1:
        raiseHttpReadError("Invalid headers received, too many `Content-Type`")
      if res.count(ContentLengthHeader) > 1:
        raiseHttpReadError("Invalid headers received, too many `Content-Length`")
      if res.count(TransferEncodingHeader) > 1:
        raiseHttpReadError("Invalid headers received, too many `Transfer-Encoding`")
      res

  waitFor req.closeWait
  return headers


proc fetchHeaders*(request: HttpClientRequestRef): Future[HttpTable] {.
     async.} =
  doAssert(request.state == HttpReqRespState.Ready,
           "Request's state is " & $request.state)
  let connection =
    try:
      await request.session.acquireConnection(request.address)
    except CancelledError as exc:
      request.setError(newHttpInterruptError())
      raise exc
    except HttpError as exc:
      request.setError(exc)
      raise exc

  request.connection = connection

  try:
    let headers = request.prepareRequest()
    request.connection.state = HttpClientConnectionState.RequestHeadersSending
    request.state = HttpReqRespState.Open
    await request.connection.writer.write(headers)
    request.connection.state = HttpClientConnectionState.RequestHeadersSent
    request.connection.state = HttpClientConnectionState.RequestBodySending
    if len(request.buffer) > 0:
      await request.connection.writer.write(request.buffer)
    request.connection.state = HttpClientConnectionState.RequestBodySent
    request.state = HttpReqRespState.Finished
  except CancelledError as exc:
    request.setError(newHttpInterruptError())
    raise exc
  except AsyncStreamError as exc:
    let error = newHttpWriteError("Could not send request headers")
    request.setError(error)
    raise error

  let resp =
    try:
      await request.getHeaderOnlyResponse()
    except CancelledError as exc:
      request.setError(newHttpInterruptError())
      raise exc
    except HttpError as exc:
      request.setError(exc)
      raise exc
  return resp

This probably isn't a common use case and worth adding to the implementation, so I'm just dropping it here in case someone finds it useful.

@cheatfate
Copy link
Collaborator

Here i want to show how to obtain response object, which can be used to obtain response headers.

import chronos/apps/http/httpclient

proc getClient() {.async.} =
  var session = HttpSessionRef.new({HttpClientFlag.NoVerifyHost,
                                    HttpClientFlag.NoVerifyServerName},
                                    maxRedirections = 10)
  var request =
    block:
      let res = HttpClientRequestRef.new(session, "https://httpbin.org/get",
                                         meth = MethodGet)
      if res.isErr():
        echo "ERROR: ", res.error()
        quit(1)
      res.get()
  # Here we obtain `response` object.
  var response = await request.send()
  echo "HTTP RESPONSE STATUS CODE = ", response.status
  echo "HTTP RESPONSE HEADES: "
  echo $response.headers
  await response.closeWait()
  await request.closeWait()
  await session.closeWait()

proc postClient() {.async.} =
  var session = HttpSessionRef.new({HttpClientFlag.NoVerifyHost,
                                    HttpClientFlag.NoVerifyServerName},
                                    maxRedirections = 10)
  var request =
    block:
      let res = HttpClientRequestRef.new(session, "https://httpbin.org/post",
                                         meth = MethodPost)
      if res.isErr():
        echo "ERROR: ", res.error()
        quit(1)
      res.get()
  request.headers.set("Content-type", "application/json")
  request.headers.set("Accept", "application/json")
  request.headers.set("Transfer-encoding", "chunked")
  var writer = await request.open()
  await writer.write("{\"data\": \"data\"}")
  await writer.finish()
  await writer.closeWait()
  # Here we obtain `response` object.
  var response = await request.finish()
  echo "HTTP RESPONSE STATUS CODE = ", response.status
  echo "HTTP RESPONSE HEADES: "
  echo $response.headers
  await response.closeWait()
  await request.closeWait()
  await session.closeWait()

when isMainModule:
  waitFor getClient()
  waitFor postClient()

As you can see there no need to dive deep into chronos internals.

@cheatfate
Copy link
Collaborator

And this is main difference between std/httpclient and chronos.httpclient because in both samples above, body has not been received yet, only response headers are obtained from the network.

@arnetheduck
Copy link
Member

@cheatfate how about we create an examples/ folder and put a few of these simple cases in there? along with https://forum.nim-lang.org/t/7964#52137 and perhaps a TCP ping/pong server

@cheatfate cheatfate added enhancement New feature or request documentation and removed enhancement New feature or request labels Mar 22, 2023
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Projects
None yet
Development

No branches or pull requests

3 participants