-
Notifications
You must be signed in to change notification settings - Fork 1
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Include job type in key names. Closes #1 Record task start + end times, processing time and wall-time. Closes #2 Rollup counter and timers into parent (shard -> namespace, namespace -> job). Closes #3 Rename iterator / splitter. Closes #5 Record job data and use to track overall status when shard / namespaces complete. Closes #6 Use transactional task enqueueing rather than named tasks #7 Provide abort mechanism (no write yet) #9 Output to cloud storage and rollup files (slice -> shard, shard -> namespace) Closes #12
- Loading branch information
1 parent
7eafe10
commit ef6d680
Showing
21 changed files
with
1,741 additions
and
972 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,93 @@ | ||
package mapper | ||
|
||
import ( | ||
"strings" | ||
"time" | ||
|
||
"google.golang.org/appengine/datastore" | ||
) | ||
|
||
type ( | ||
// common contains properties that are common across all | ||
// mapper entities (job, iterator, namespace and shard) | ||
common struct { | ||
// Counters holds the task counters map | ||
Counters Counters `datastore:"-"` | ||
|
||
// Active indicates if this task is still active | ||
Active bool `datastore:"active,noindex"` | ||
|
||
// Count is the number of records processed | ||
Count int64 `datastore:"count,noindex"` | ||
|
||
// Started is when the task began | ||
Started time.Time `datastore:"started"` | ||
|
||
// Updated is when the task was last updated | ||
Updated time.Time `datastore:"updated"` | ||
|
||
// ProcessTime is the time that the task spent executing | ||
ProcessTime time.Duration `datastore:"process_time,noindex"` | ||
|
||
// WallTime is the wall-time that the task takes | ||
WallTime time.Duration `datastore:"wall_time,noindex"` | ||
|
||
// private fields used by local instance | ||
id string | ||
queue string | ||
startTime time.Time | ||
} | ||
) | ||
|
||
func (c *common) start() { | ||
c.Active = true | ||
c.Counters = NewCounters() | ||
c.Count = 0 | ||
c.Started = getTime() | ||
c.Updated = c.Started | ||
c.startTime = c.Started | ||
} | ||
|
||
func (c *common) complete() { | ||
c.Active = false | ||
c.Updated = getTime() | ||
c.WallTime = c.Updated.Sub(c.Started) | ||
} | ||
|
||
func (c *common) rollup(r common) { | ||
c.Count += r.Count | ||
c.ProcessTime += r.ProcessTime | ||
c.Counters.Add(r.Counters) | ||
} | ||
|
||
/* datastore */ | ||
func (c *common) Load(props []datastore.Property) error { | ||
datastore.LoadStruct(c, props) | ||
|
||
c.Counters = make(map[string]int64) | ||
for _, prop := range props { | ||
if strings.HasPrefix(prop.Name, "counters.") { | ||
key := prop.Name[9:len(prop.Name)] | ||
c.Counters[key] = prop.Value.(int64) | ||
} | ||
} | ||
|
||
c.startTime = getTime() | ||
|
||
return nil | ||
} | ||
|
||
func (c *common) Save() ([]datastore.Property, error) { | ||
c.ProcessTime += getTime().Sub(c.startTime) | ||
|
||
props, err := datastore.SaveStruct(c) | ||
if err != nil { | ||
return nil, err | ||
} | ||
|
||
for key, value := range c.Counters { | ||
props = append(props, datastore.Property{Name: "counters." + key, Value: value, NoIndex: true, Multiple: false}) | ||
} | ||
|
||
return props, nil | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1 +1,5 @@ | ||
package mapper // import "github.com/captaincodeman/datastore-mapper" | ||
|
||
/* | ||
mapper is a library to iterate over appengine Datastore entities | ||
*/ |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,163 @@ | ||
package mapper | ||
|
||
import ( | ||
"net/http" | ||
|
||
"golang.org/x/net/context" | ||
"google.golang.org/appengine" | ||
"google.golang.org/appengine/datastore" | ||
"google.golang.org/appengine/log" | ||
) | ||
|
||
const ( | ||
iteratorURL = "/iterate" | ||
iteratorCompleteURL = iteratorURL + "/complete" | ||
) | ||
|
||
func init() { | ||
Server.HandleFunc(iteratorURL, iteratorHandler) | ||
Server.HandleFunc(iteratorCompleteURL, iteratorCompleteHandler) | ||
} | ||
|
||
func iteratorHandler(w http.ResponseWriter, r *http.Request) { | ||
if r.Method != "POST" { | ||
return | ||
} | ||
|
||
c := appengine.NewContext(r) | ||
|
||
id, seq, queue, _ := ParseLock(r) | ||
log.Infof(c, "iterator %s %d", id, seq) | ||
|
||
k := datastore.NewKey(c, config.DatastorePrefix+iteratorKind, id, 0, nil) | ||
it := new(iterator) | ||
|
||
if err := GetLock(c, k, it, seq); err != nil { | ||
if serr, ok := err.(*LockError); ok { | ||
// for locking errors, the error gives us the response to use | ||
w.WriteHeader(serr.Response) | ||
w.Write([]byte(serr.Error())) | ||
} else { | ||
w.WriteHeader(http.StatusInternalServerError) | ||
w.Write([]byte(err.Error())) | ||
} | ||
log.Errorf(c, "error %s", err.Error()) | ||
return | ||
} | ||
|
||
it.id = id | ||
it.queue = queue | ||
|
||
j, err := getJob(c, id) | ||
if err != nil { | ||
w.WriteHeader(http.StatusInternalServerError) | ||
w.Write([]byte(err.Error())) | ||
ClearLock(c, k, it, false) | ||
log.Errorf(c, "error %s", err.Error()) | ||
return | ||
} | ||
|
||
if j.Abort { | ||
w.WriteHeader(http.StatusOK) | ||
return | ||
} | ||
|
||
it.job = j | ||
|
||
completed, err := it.iterate(c) | ||
if err != nil { | ||
// this will cause a task retry | ||
w.WriteHeader(http.StatusInternalServerError) | ||
w.Write([]byte(err.Error())) | ||
ClearLock(c, k, it, false) | ||
log.Errorf(c, "error %s", err.Error()) | ||
return | ||
} | ||
|
||
var url string | ||
if completed { | ||
// schedule completion | ||
url = config.BasePath + iteratorCompleteURL | ||
} else { | ||
// schedule continuation | ||
url = config.BasePath + iteratorURL | ||
} | ||
_, err = ScheduleLock(c, k, it, url, nil, queue) | ||
if err != nil { | ||
// this will cause a task retry | ||
w.WriteHeader(http.StatusInternalServerError) | ||
w.Write([]byte(err.Error())) | ||
ClearLock(c, k, it, false) | ||
} else { | ||
w.WriteHeader(http.StatusOK) | ||
} | ||
} | ||
|
||
func iteratorCompleteHandler(w http.ResponseWriter, r *http.Request) { | ||
if r.Method != "POST" { | ||
return | ||
} | ||
|
||
c := appengine.NewContext(r) | ||
|
||
id, seq, queue, _ := ParseLock(r) | ||
log.Infof(c, "iterator complete %s %d", id, seq) | ||
|
||
k := datastore.NewKey(c, config.DatastorePrefix+iteratorKind, id, 0, nil) | ||
it := new(iterator) | ||
|
||
if err := GetLock(c, k, it, seq); err != nil { | ||
if serr, ok := err.(*LockError); ok { | ||
// for locking errors, the error gives us the response to use | ||
w.WriteHeader(serr.Response) | ||
w.Write([]byte(serr.Error())) | ||
} else { | ||
w.WriteHeader(http.StatusInternalServerError) | ||
w.Write([]byte(err.Error())) | ||
} | ||
log.Errorf(c, "error %s", err.Error()) | ||
return | ||
} | ||
|
||
it.id = id | ||
it.queue = queue | ||
|
||
// mark iterator as complete | ||
it.complete() | ||
it.RequestID = "" | ||
|
||
// update iterator status and job within a transaction | ||
jk := datastore.NewKey(c, config.DatastorePrefix+jobKind, id, 0, nil) | ||
j := new(jobState) | ||
err := storage.RunInTransaction(c, func(tc context.Context) error { | ||
if err := storage.Get(tc, jk, j); err != nil { | ||
return err | ||
} | ||
if j.Abort { | ||
return nil | ||
} | ||
|
||
j.NamespacesTotal += int(it.Count) | ||
j.Iterating = false | ||
// only the iterator walltime is rolled up into the job counts | ||
j.WallTime += it.WallTime | ||
|
||
if _, err := storage.Put(tc, k, it); err != nil { | ||
return err | ||
} | ||
if _, err := storage.Put(tc, jk, j); err != nil { | ||
return err | ||
} | ||
return nil | ||
}, &datastore.TransactionOptions{XG: true, Attempts: attempts}) | ||
|
||
if err != nil { | ||
w.WriteHeader(http.StatusInternalServerError) | ||
w.Write([]byte(err.Error())) | ||
ClearLock(c, k, it, false) | ||
log.Errorf(c, "error %s", err.Error()) | ||
return | ||
} | ||
|
||
w.WriteHeader(http.StatusOK) | ||
} |
Oops, something went wrong.