-
Notifications
You must be signed in to change notification settings - Fork 0
/
app.go
179 lines (156 loc) · 5.47 KB
/
app.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
package rdiff
import (
"crypto/md5" // nolint
"encoding/gob"
"errors"
"fmt"
"io"
"math"
"os"
)
const (
// DefaultBlockSize is the default block size value, in bytes, used by the system
DefaultBlockSize = 700
// MaxBlockSize is the max block size value, in bytes, used by the system
MaxBlockSize = 1 << 17
)
// App is the application layer of the RDiff service.
// It exposes the public API and allows for IO interactions.
type App struct {
diffEngine *rDiff
}
// New constructs the RDiff app instance and returns a pointer to it.
// It accepts a blockSize as input, representing the size, in bytes, for splitting the target in blocks,
// in order to compute the target's signature.
// A blockSize <=0 means the size, in bytes, ii computed dynamically.
func New(blockSize int) *App {
return &App{
// nolint
diffEngine: newRDiff(blockSize, newAdler32RollingHash(), md5.New()),
}
}
// Signature computes the signature of a target file(targetFilePath) and writes it to an output file(outputFilePath)
// The target file(targetFileName) must exist, otherwise it returns an appropriate non-nil error.
// If the output file(outputFilePath) already exists, it returns an appropriate non-nil error.
// The content written to outputFilePath is serialized using gob encoding.
func (a *App) Signature(targetFilePath string, signatureFilePath string) error {
targetFile, err := os.Open(targetFilePath)
if err != nil {
return err
}
tfInfo, err := targetFile.Stat()
if err != nil {
return err
}
targetFileSize := tfInfo.Size()
if targetFileSize <= 0 {
return errors.New("the target file is empty")
}
a.diffEngine.blockSize, err = decideBlockSize(a.diffEngine.blockSize, targetFileSize)
if err != nil {
return err
}
signatureFile, err := os.OpenFile(signatureFilePath, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0666)
if err != nil {
return err
}
err = a.signature(targetFile, signatureFile)
err1 := targetFile.Close()
err2 := signatureFile.Close()
return errors.Join(err, err1, err2)
}
// Delta computes the instruction list(operations list) in order for the target
// to be able to update its content to match the source.
// The signature file(signatureFilePath) and the source file(sourceFilePath) must exist,
// otherwise a non-nil error is returned.
// The delta file(deltaFilePath) must not exist, otherwise a non-nil error is returned.
// The content written to deltaFilePath is serialized using gob encoding.
func (a *App) Delta(signatureFilePath string, sourceFilePath string, deltaFilePath string) error {
signatureFile, err := os.Open(signatureFilePath)
if err != nil {
return err
}
sourceFile, err := os.Open(sourceFilePath)
if err != nil {
return err
}
deltaFile, err := os.OpenFile(deltaFilePath, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0666)
if err != nil {
return err
}
err = a.delta(signatureFile, sourceFile, deltaFile)
err1 := signatureFile.Close()
err2 := sourceFile.Close()
err3 := deltaFile.Close()
return errors.Join(err, err1, err2, err3)
}
// delta is the lower layer that performs the delta computation and data serialization.
func (a *App) delta(signature, source io.Reader, output io.Writer) error {
var blockList []Block
err := gob.NewDecoder(signature).Decode(&blockList)
if err != nil {
return err
}
delta, err := a.diffEngine.ComputeDelta(source, blockList)
if err != nil {
return err
}
return gob.NewEncoder(output).Encode(delta)
}
// signature is the lower layer that performs the signature computation and data serialization.
func (a *App) signature(target io.Reader, output io.Writer) error {
signature, err := a.diffEngine.ComputeSignature(target)
if err != nil {
return err
}
return gob.NewEncoder(output).Encode(signature)
}
// computeDynamicBlockSize is the actual rsync algorithm for computing the dynamic block size, based on the file length.
// it does some computation to evenly distribute the blockSize according to fLen size.
func computeDynamicBlockSize(fLen int64) int64 {
if fLen <= DefaultBlockSize*DefaultBlockSize {
return DefaultBlockSize
}
var c, l, cnt int64
for c, l, cnt = 1, fLen, 0; cnt < l<<2; c, l, cnt = c<<1, l>>2, cnt+1 {
}
if c < 0 || c >= MaxBlockSize {
return MaxBlockSize
}
var blSize int64
for c >= 8 {
blSize |= c
if fLen < blSize*blSize {
blSize &= ^c
}
c >>= 1
}
return max(blSize, DefaultBlockSize)
}
// decideBlockSize make the logical decision against splitting a fileSize in blockSizes
// if blockSize <= 0 a new blocSize is dynamically computed and returned
// if blockSize > 0 it is validated against splitting the fileSize in at least 2 parts and returns a non-nil error if
// it's not able to do so (ex: round(fileSize/blockSize < 2)
func decideBlockSize(blockSize int, fileSize int64) (int, error) {
// if provided blockSize, validate against min nr of chunks
if blockSize > 0 {
nrOfChunks := int(math.Ceil(float64(fileSize) / float64(blockSize)))
// check that the diff makes sense - at least 2 blocks for the signature file
if nrOfChunks < 2 {
return 0, fmt.Errorf(
"the current blockSize(%v) doesn't allow splitting target(size: %v) in at least 2 blocks/chunks",
blockSize,
fileSize,
)
}
return blockSize, nil
}
// if not provided blockSize, switch to dynamic and adjust if block size is too small
blockSize = int(computeDynamicBlockSize(fileSize))
nrOfChunks := int(math.Ceil(float64(fileSize) / float64(blockSize)))
// ensure a min of 2 chunks for the dynamically computed size
if nrOfChunks < 2 {
blockSize = int(math.Floor(float64(fileSize) / 2))
}
return blockSize, nil
}