Skip to content

Commit

Permalink
Merge pull request #56 from richardlehane/develop
Browse files Browse the repository at this point in the history
Develop
  • Loading branch information
richardlehane committed Nov 6, 2015
2 parents af60cab + 98b8948 commit d322ef9
Show file tree
Hide file tree
Showing 17 changed files with 312 additions and 136 deletions.
15 changes: 11 additions & 4 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ Key features are:

## Version

1.4.0
1.4.1

[![Build Status](https://travis-ci.org/richardlehane/siegfried.png?branch=master)](https://travis-ci.org/richardlehane/siegfried) [![GoDoc](https://godoc.org/github.com/richardlehane/siegfried?status.svg)](https://godoc.org/github.com/richardlehane/siegfried)

Expand All @@ -29,18 +29,20 @@ Key features are:
sf -csv file.ext | DIR // Output CSV rather than YAML
sf -json file.ext | DIR // Output JSON rather than YAML
sf -droid file.ext | DIR // Output DROID CSV rather than YAML
sf -known file.ext | DIR // Output list of recognised files
sf -unknown file.ext | DIR // Output list of unrecognised files
sf - // Read list of files piped to stdin
sf -nr DIR // Don't scan subdirectories
sf -z file.zip | DIR // Decompress and scan zip, tar, gzip, warc, arc
sf -hash md5 file.ext | DIR // Calculate md5, sha1, sha256, sha512, or crc hash
sf -sig custom.sig file.ext // Use a custom signature file
sf -home c:\junk -sig custom.sig file.ext // Use a custom home directory
sf -debug file.ext // Scan in debug mode
sf -slow file.ext | DIR // Report slow signatures
sf -version // Display version information
sf -serve hostname:port // Server mode
sf -log [comma-sep opts] file.ext | DIR // Log errors etc. to stderr (default) or stdout
sf -log e,w file.ext | DIR // Log errors and warnings to stderr
sf -log u,o file.ext | DIR // Log unknowns to stdout
sf -log d,s file.ext | DIR // Log debugging and slow messages to stderr
sf -log p DIR > results.yaml // Log progress while redirecting results


![Usage](usage.gif)
Expand Down Expand Up @@ -75,6 +77,11 @@ Download a pre-built binary from the [releases page](https://github.com/richardl
sf -update

## Recent Changes
### Version 1.4.1
- **-log replaces -debug, -slow, -unknown and -known flags** (see usage above)
- highlight empty file/stream with error and warning
- negative text match overrides extension-only plain text match

### Version 1.4.0 (31/10/2015)
- new MIME matcher; requested by [Dragan Espenschied](https://github.com/richardlehane/siegfried/issues/55)
- support warc continuations
Expand Down
12 changes: 10 additions & 2 deletions cmd/sf/fpr.go
Original file line number Diff line number Diff line change
Expand Up @@ -38,13 +38,18 @@ func reply(s string) []byte {
}

func fpridentify(s *siegfried.Siegfried, path string) []byte {
lg.set(path)
fi, err := os.Open(path)
defer fi.Close()
if err != nil {
lg.err(err)
lg.reset()
return reply("error: failed to open " + path + "; got " + err.Error())
}
c, err := s.Identify(fi, path, "")
if err != nil {
lg.err(err)
if c == nil {
lg.reset()
return reply("error: failed to scan " + path + "; got " + err.Error())
}
var ids []string
Expand All @@ -54,7 +59,9 @@ func fpridentify(s *siegfried.Siegfried, path string) []byte {
if !i.Known() {
warn = i.(*pronom.Identification).Warning
}
lg.id(i)
}
lg.reset()
switch len(ids) {
case 0:
return reply("error: scanning " + path + ": no puids returned")
Expand Down Expand Up @@ -90,8 +97,9 @@ func serveFpr(addr string, s *siegfried.Siegfried) {
l, err := conn.Read(buf)
if err != nil {
conn.Write([]byte("error reading from connection: " + err.Error()))
} else {
conn.Write(fpridentify(s, string(buf[:l])))
}
conn.Write(fpridentify(s, string(buf[:l])))
conn.Close()
}
}
135 changes: 135 additions & 0 deletions cmd/sf/logger.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,135 @@
// Copyright 2015 Richard Lehane. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
"fmt"
"io"
"os"
"strings"

"github.com/richardlehane/siegfried/config"
"github.com/richardlehane/siegfried/pkg/core"
)

var lg *logger

type logger struct {
progress bool
e bool
warn bool
debug bool
slow bool
unknown bool
known bool
w io.Writer
// mutate
path string
u bool
fp bool // file name already printed
}

func newLogger(opts string) error {
l := &logger{w: os.Stderr}
for _, o := range strings.Split(opts, ",") {
switch o {
case "stderr":
case "stdout", "out", "o":
l.w = os.Stdout
case "progress", "p":
l.progress = true
case "error", "err", "e":
l.e = true
case "warning", "warn", "w":
l.warn = true
case "debug", "d":
l.debug, l.progress = true, true
config.SetDebug()
case "slow", "s":
l.slow, l.progress = true, true
config.SetSlow()
case "unknown", "u":
l.unknown = true
case "known", "k":
l.known = true
default:
return fmt.Errorf("unknown -log input %s; expect be comma-separated list of stdout,out,o,progress,p,error,err,e,warning,warn,w,debug,d,slow,s,unknown,u,known,k", opts)
}
}
if config.Debug() || config.Slow() {
config.SetOut(l.w)
}
lg = l
return nil
}

var (
fileString = "[FILE]"
errString = "[ERROR]"
warnString = "[WARN]"
)

func (l *logger) printFile() {
if !l.fp {
fmt.Fprintf(l.w, "%s %s\n", fileString, l.path)
l.fp = true
}
}

func (l *logger) set(path string) {
if l == nil {
return
}
l.path = path
if l.progress {
l.printFile()
}
}

func (l *logger) err(err error) {
if l != nil && l.e && err != nil {
l.printFile()
fmt.Fprintf(l.w, "%s %v\n", errString, err)
}
}

func (l *logger) id(i core.Identification) {
if l == nil {
return
}
if (l.unknown || l.known) && i.Known() {
l.u = true
}
if l.warn {
if w := i.Warn(); w != "" {
l.printFile()
fmt.Fprintf(l.w, "%s %s\n", warnString, w)
}
}
if l.slow || l.debug {
fmt.Fprintf(l.w, "matched: %s\n", i.String())
}
}

func (l *logger) reset() {
if l == nil {
return
}
if (l.known && l.u) || (l.unknown && !l.u) {
fmt.Fprintln(l.w, l.path)
}
l.u, l.fp = false, false
l.path = ""
}
12 changes: 11 additions & 1 deletion cmd/sf/serve.go
Original file line number Diff line number Diff line change
Expand Up @@ -98,22 +98,32 @@ func identify(s *siegfried.Siegfried) func(w http.ResponseWriter, r *http.Reques
w.Header().Set("Content-Type", mime)
wr.writeHead(s)
c, err := s.Identify(f, h.Filename, "")
lg.set(h.Filename)
lg.err(err)
if c == nil {
wr.writeFile(h.Filename, sz, mod, nil, fmt.Errorf("failed to identify %s, got: %v", h.Filename, err), nil)
wr.writeFile(h.Filename, sz, mod, nil, err, nil)
lg.reset()
return
}
wr.writeFile(h.Filename, sz, mod, nil, err, idChan(c))
wr.writeTail()
lg.reset()
return
} else {
path, err := decodePath(r.URL.Path)
if err != nil {
handleErr(w, http.StatusNotFound, err)
lg.set(path)
lg.err(err)
lg.reset()
return
}
info, err := os.Stat(path)
if err != nil {
handleErr(w, http.StatusNotFound, err)
lg.set(path)
lg.err(err)
lg.reset()
return
}
w.Header().Set("Content-Type", mime)
Expand Down
Loading

0 comments on commit d322ef9

Please sign in to comment.