# Git LFS Changelog
+## 2.3.1 (27 September, 2017)
+
+### Features
+
+* add support for SSH_ASKPASS #2609 (@technoweenie)
+* `git lfs migrate --verbose` option #2610 (@technoweenie)
+* Support standalone custom transfer based on API URL prefix match #2590 (@sprohaska)
+
+### Bugs
+
+* Improve invalid URL error messages #2614 (@technoweenie)
+* Fix double counting progress bug #2608 (@technoweenie)
+* trim whitespace from GIT_ASKPASS provided passwords #2607 (@technoweenie)
+* remove mmap usage in Packfile reader #2600 (@technoweenie)
+* `git lfs clone`: don't fetch for unborn repositories #2598 (@shiftkey)
+
+### Misc
+
+* Windows Installer fixes:
+ * Show proper icon in add/remove programs list #2585 (@shiftkey)
+ * Make the Inno Setup installer script explicitly check for the binaries #2588 (@sschuberth)
+ * Improve compile-win-installer-unsigned.bat a bit #2586 (@sschuberth)
+* Update migrate docs example for multiple file types #2596 (@technoweenie)
+
## 2.3.0 (14 September, 2017)
Git LFS v2.3.0 includes performance optimizations for the `git-lfs-migrate(1)`
}
var totalBytes int64
+ var pointers []*lfs.WrappedPointer
meter := progress.NewMeter(progress.WithOSEnv(cfg.Os))
- singleCheckout := newSingleCheckout()
chgitscanner := lfs.NewGitScanner(func(p *lfs.WrappedPointer, err error) {
if err != nil {
LoggedError(err, "Scanner error: %s", err)
totalBytes += p.Size
meter.Add(p.Size)
meter.StartTransfer(p.Name)
-
- singleCheckout.Run(p)
-
- // not strictly correct (parallel) but we don't have a callback & it's just local
- // plus only 1 slot in channel so it'll block & be close
- meter.TransferBytes("checkout", p.Name, p.Size, totalBytes, int(p.Size))
- meter.FinishTransfer(p.Name)
+ pointers = append(pointers, p)
})
chgitscanner.Filter = filepathfilter.New(rootedPaths(args), nil)
if err := chgitscanner.ScanTree(ref.Sha); err != nil {
ExitWithError(err)
}
+ chgitscanner.Close()
+ singleCheckout := newSingleCheckout()
meter.Start()
- chgitscanner.Close()
+ for _, p := range pointers {
+ singleCheckout.Run(p)
+
+ // not strictly correct (parallel) but we don't have a callback & it's just local
+ // plus only 1 slot in channel so it'll block & be close
+ meter.TransferBytes("checkout", p.Name, p.Size, totalBytes, int(p.Size))
+ meter.FinishTransfer(p.Name)
+ }
+
meter.Finish()
singleCheckout.Close()
}
remote = "origin"
}
- includeArg, excludeArg := getIncludeExcludeArgs(cmd)
- filter := buildFilepathFilter(cfg, includeArg, excludeArg)
- if cloneFlags.NoCheckout || cloneFlags.Bare {
- // If --no-checkout or --bare then we shouldn't check out, just fetch instead
- cfg.CurrentRemote = remote
- fetchRef("HEAD", filter)
- } else {
- pull(remote, filter)
- err := postCloneSubmodules(args)
- if err != nil {
- Exit("Error performing 'git lfs pull' for submodules: %v", err)
+ if ref, err := git.CurrentRef(); err == nil {
+ includeArg, excludeArg := getIncludeExcludeArgs(cmd)
+ filter := buildFilepathFilter(cfg, includeArg, excludeArg)
+ if cloneFlags.NoCheckout || cloneFlags.Bare {
+ // If --no-checkout or --bare then we shouldn't check out, just fetch instead
+ cfg.CurrentRemote = remote
+ fetchRef(ref.Name, filter)
+ } else {
+ pull(remote, filter)
+ err := postCloneSubmodules(args)
+ if err != nil {
+ Exit("Error performing 'git lfs pull' for submodules: %v", err)
+ }
}
}
}
ready, pointers, meter := readyAndMissingPointers(allpointers, filter)
- q := newDownloadQueue(getTransferManifest(), cfg.CurrentRemote, tq.WithProgress(meter))
+ q := newDownloadQueue(
+ getTransferManifestOperationRemote("download", cfg.CurrentRemote),
+ cfg.CurrentRemote, tq.WithProgress(meter),
+ )
if out != nil {
// If we already have it, or it won't be fetched
available := make(chan *tq.Transfer)
if supportsDelay {
- q = tq.NewTransferQueue(tq.Download, getTransferManifest(), cfg.CurrentRemote)
+ q = tq.NewTransferQueue(
+ tq.Download,
+ getTransferManifestOperationRemote("download", cfg.CurrentRemote),
+ cfg.CurrentRemote,
+ )
go infiniteTransferBuffer(q, available)
}
// migrateEverything indicates the presence of the --everything flag,
// and instructs 'git lfs migrate' to migrate all local references.
migrateEverything bool
+
+ // migrateVerbose enables verbose logging
+ migrateVerbose bool
)
// migrate takes the given command and arguments, *odb.ObjectDatabase, as well
Exclude: exclude,
UpdateRefs: opts.UpdateRefs,
+ Verbose: opts.Verbose,
BlobFn: opts.BlobFn,
TreeCallbackFn: opts.TreeCallbackFn,
info.Flags().StringVar(&migrateInfoUnitFmt, "unit", "", "--unit=<unit>")
importCmd := NewCommand("import", migrateImportCommand)
+ importCmd.Flags().BoolVar(&migrateVerbose, "verbose", false, "Verbose logging")
RegisterCommand("migrate", nil, func(cmd *cobra.Command) {
cmd.PersistentFlags().StringVarP(&includeArg, "include", "I", "", "Include a list of paths")
func migrateImportCommand(cmd *cobra.Command, args []string) {
l := log.NewLogger(os.Stderr)
+ defer l.Close()
db, err := getObjectDatabase()
if err != nil {
ExitWithError(err)
}
+ defer db.Close()
+
rewriter := getHistoryRewriter(cmd, db, l)
tracked := trackedFromFilter(rewriter.Filter())
exts := tools.NewOrderedSet()
migrate(args, rewriter, l, &githistory.RewriteOptions{
+ Verbose: migrateVerbose,
BlobFn: func(path string, b *odb.Blob) (*odb.Blob, error) {
if filepath.Base(path) == ".gitattributes" {
return b, nil
UpdateRefs: true,
})
+ // Only perform `git-checkout(1) -f` if the repository is
+ // non-bare.
if bare, _ := git.IsBare(); !bare {
- // Only perform `git-checkout(1) -f` if the repository is
- // non-bare.
- if err := git.Checkout("", nil, true); err != nil {
+ t := l.Waiter("migrate: checkout")
+ err := git.Checkout("", nil, true)
+ t.Complete()
+
+ if err != nil {
ExitWithError(err)
}
}
if err != nil {
ExitWithError(err)
}
+ defer db.Close()
+
rewriter := getHistoryRewriter(cmd, db, l)
exts := make(map[string]*MigrateInfoEntry)
return b, nil
},
})
+ l.Close()
entries := EntriesBySize(MapToEntries(exts))
entries = removeEmptyEntries(entries)
var verifywait sync.WaitGroup
if verifyRemote {
- verifyQueue = newDownloadCheckQueue(getTransferManifest(), fetchPruneConfig.PruneRemoteName)
+ verifyQueue = newDownloadCheckQueue(
+ getTransferManifestOperationRemote("download", fetchPruneConfig.PruneRemoteName),
+ fetchPruneConfig.PruneRemoteName,
+ )
verifiedObjects = tools.NewStringSetWithCapacity(len(localObjects) / 2)
// this channel is filled with oids for which Check() succeeded & Transfer() was called
}
meter.Add(p.Size)
- meter.StartTransfer(p.Name)
tracerx.Printf("fetch %v [%v]", p.Name, p.Oid)
pointers.Add(p)
q.Add(downloadTransfer(p))
ManPages = make(map[string]string, 20)
cfg = config.Config
- tqManifest *tq.Manifest
+ tqManifest = make(map[string]*tq.Manifest)
apiClient *lfsapi.Client
global sync.Mutex
// getTransferManifest builds a tq.Manifest from the global os and git
// environments.
func getTransferManifest() *tq.Manifest {
+ return getTransferManifestOperationRemote("", "")
+}
+
+// getTransferManifestOperationRemote builds a tq.Manifest from the global os
+// and git environments and operation-specific and remote-specific settings.
+// Operation must be "download", "upload", or the empty string.
+func getTransferManifestOperationRemote(operation, remote string) *tq.Manifest {
c := getAPIClient()
global.Lock()
defer global.Unlock()
- if tqManifest == nil {
- tqManifest = tq.NewManifestWithClient(c)
+ k := fmt.Sprintf("%s.%s", operation, remote)
+ if tqManifest[k] == nil {
+ tqManifest[k] = tq.NewManifestClientOperationRemote(c, operation, remote)
}
- return tqManifest
+ return tqManifest[k]
}
func getAPIClient() *lfsapi.Client {
ctx := &uploadContext{
Remote: remote,
- Manifest: getTransferManifest(),
+ Manifest: getTransferManifestOperationRemote("upload", remote),
DryRun: dryRun,
uploadedOids: tools.NewStringSet(),
ourLocks: make(map[string]locking.Lock),
ctx.tq = newUploadQueue(ctx.Manifest, ctx.Remote, tq.WithProgress(ctx.meter), tq.DryRun(ctx.DryRun))
ctx.committerName, ctx.committerEmail = cfg.CurrentCommitter()
+ // Do not check locks for standalone transfer, because there is no LFS
+ // server to ask.
+ if ctx.Manifest.IsStandaloneTransfer() {
+ ctx.lockVerifyState = verifyStateDisabled
+ return ctx
+ }
+
ourLocks, theirLocks, verifyState := verifyLocks(remote)
ctx.lockVerifyState = verifyState
for _, l := range theirLocks {
// given "endpoint". If no state has been explicitly set, an "unknown" state
// will be returned instead.
func getVerifyStateFor(endpoint lfsapi.Endpoint) verifyState {
- if v, _ := cfg.Git.Get("lfs.standalonetransferagent"); len(v) > 0 {
- // When using a standalone custom transfer agent there is no LFS server.
- return verifyStateDisabled
- }
-
uc := config.NewURLConfig(cfg.Git)
v, ok := uc.Get("lfs", endpoint.Url, "locksverify")
)
const (
- Version = "2.3.0"
+ Version = "2.3.1"
)
func init() {
+git-lfs (2.3.1) stable; urgency=low
+
+ * New upstream version
+
+ -- Rick Olson <technoweenie@gmail.com> Wed, 26 Sep 2017 14:29:00 +0000
+
git-lfs (2.3.0) stable; urgency=low
* New upstream version
In this case it's possible to use the custom transfer agent directly,
without querying the server, by using the following config option:
-* `lfs.standalonetransferagent`
+* `lfs.standalonetransferagent`, `lfs.<url>.standalonetransferagent`
- Allows the specified custom transfer agent to be used directly
- for transferring files, without asking the server how the transfers
- should be made. The custom transfer agent has to be defined in a
- `lfs.customtransfer.<name>` settings group.
+ Specifies a custom transfer agent to be used if the API server URL matches as
+ in `git config --get-urlmatch lfs.standalonetransferagent <apiurl>`.
+ `git-lfs` will not contact the API server. It instead sets stage 2 transfer
+ actions to `null`. `lfs.<url>.standalonetransferagent` can be used to
+ configure a custom transfer agent for individual remotes.
+ `lfs.standalonetransferagent` unconditionally configures a custom transfer
+ agent for all remotes. The custom transfer agent must be specified in
+ a `lfs.customtransfer.<name>` settings group.
## Defining a Custom Transfer Type
The message will look like this:
```json
-{ "event": "init", "operation": "download", "concurrent": true, "concurrenttransfers": 3 }
+{ "event": "init", "operation": "download", "remote": "origin", "concurrent": true, "concurrenttransfers": 3 }
```
* `event`: Always `init` to identify this message
* `operation`: will be `upload` or `download` depending on transfer direction
+* `remote`: The Git remote. It can be a remote name like `origin` or an URL
+ like `ssh://git.example.com//path/to/repo`. A standalone transfer agent can
+ use it to determine the location of remote files.
* `concurrent`: reflects the value of `lfs.customtransfer.<name>.concurrent`, in
case the process needs to know
* `concurrenttransfers`: reflects the value of `lfs.concurrenttransfers`, for if
conventions, but can be interpreted however the custom transfer agent wishes
(this is an NFS example, but it doesn't even have to be an URL). Generally,
`href` will give the primary connection details, with `header` containing any
- miscellaneous information needed.
+ miscellaneous information needed. `action` is `null` for standalone transfer
+ agents.
The transfer process should post one or more [progress messages](#progress) and
then a final completion message as follows:
conventions, but can be interpreted however the custom transfer agent wishes
(this is an NFS example, but it doesn't even have to be an URL). Generally,
`href` will give the primary connection details, with `header` containing any
- miscellaneous information needed.
+ miscellaneous information needed. `action` is `null` for standalone transfer
+ agents.
Note there is no file path included in the download request; the transfer
process should create a file itself and return the path in the final response
### IMPORT
The 'import' mode migrates large objects present in the Git history to pointer
-files tracked and stored with Git LFS.
+files tracked and stored with Git LFS. It supports all the core 'migrate'
+options and these additional ones:
+
+* `--verbose`
+ Print the commit oid and filename of migrated files to STDOUT.
If `--include` or `--exclude` (`-I`, `-X`, respectively) are given, the
.gitattributes will be modified to include any new filepath patterns as given by
You can configure Git LFS to only migrate tree entries whose pathspec matches
the include glob and does not match the exclude glob, to reduce total migration
-time or to only migrate part of your repo.
+time or to only migrate part of your repo. Specify multiple patterns using the
+comma as the delimiter.
Pattern matching is done as given to be functionally equivalent to pattern
matching as in .gitattributes.
Now, you can run `git lfs migrate import` to convert some file types to LFS:
```
-$ git lfs migrate import --include="*.mp3" --include="*.psd"
+$ git lfs migrate import --include="*.mp3,*.psd"
migrate: Fetching remote refs: ..., done
migrate: Sorting commits: ..., done
migrate: Rewriting commits: 100% (1/1), done
$ git lfs migrate import --include-ref=master --include="*.zip"
# Convert all zip files in every local branch
-$ git lfs migrate info --everything --include="*.zip"
+$ git lfs migrate import --everything --include="*.zip"
```
Note: This will require a force push to any existing Git remotes.
var update *Update
for update = range task.Updates() {
- if logAll || l.throttle == 0 || update.At.After(last.Add(l.throttle)) {
+ if logAll || l.throttle == 0 || !update.Throttled(last.Add(l.throttle)) {
l.logLine(update.S)
last = update.At
}
task := make(chan *Update)
go func() {
- task <- &Update{"first", time.Now()}
- task <- &Update{"second", time.Now()}
+ task <- &Update{"first", time.Now(), false}
+ task <- &Update{"second", time.Now(), false}
close(task)
}()
t1 := make(chan *Update)
go func() {
- t1 <- &Update{"first", time.Now()}
- t1 <- &Update{"second", time.Now()}
+ t1 <- &Update{"first", time.Now(), false}
+ t1 <- &Update{"second", time.Now(), false}
close(t1)
}()
t2 := make(chan *Update)
go func() {
- t2 <- &Update{"third", time.Now()}
- t2 <- &Update{"fourth", time.Now()}
+ t2 <- &Update{"third", time.Now(), false}
+ t2 <- &Update{"fourth", time.Now(), false}
close(t2)
}()
l.widthFn = func() int { return 0 }
l.enqueue(ChanTask(t1))
- t1 <- &Update{"first", time.Now()}
+ t1 <- &Update{"first", time.Now(), false}
l.enqueue(ChanTask(t2))
close(t1)
- t2 <- &Update{"second", time.Now()}
+ t2 <- &Update{"second", time.Now(), false}
close(t2)
l.Close()
go func() {
start := time.Now()
- t1 <- &Update{"first", start} // t = 0 ms, throttle was open
- t1 <- &Update{"second", start.Add(10 * time.Millisecond)} // t = 10+ε ms, throttle is closed
- t1 <- &Update{"third", start.Add(20 * time.Millisecond)} // t = 20+ε ms, throttle was open
- close(t1) // t = 20+2ε ms, throttle is closed
+ t1 <- &Update{"first", start, false} // t = 0 ms, throttle was open
+ t1 <- &Update{"forced", start.Add(10 * time.Millisecond), true} // t = 10+ε ms, throttle is closed
+ t1 <- &Update{"second", start.Add(10 * time.Millisecond), false} // t = 10+ε ms, throttle is closed
+ t1 <- &Update{"third", start.Add(26 * time.Millisecond), false} // t = 20+ε ms, throttle was open
+ close(t1) // t = 20+2ε ms, throttle is closed
}()
l := NewLogger(&buf)
assert.Equal(t, strings.Join([]string{
"first\r",
+ "forced\r",
"third\r",
"third, done\n",
}, ""), buf.String())
go func() {
start := time.Now()
- t1 <- &Update{"first", start} // t = 0 ms, throttle was open
- t1 <- &Update{"second", start.Add(10 * time.Millisecond)} // t = 10+ε ms, throttle is closed
- close(t1) // t = 10+2ε ms, throttle is closed
+ t1 <- &Update{"first", start, false} // t = 0 ms, throttle was open
+ t1 <- &Update{"second", start.Add(10 * time.Millisecond), false} // t = 10+ε ms, throttle is closed
+ close(t1) // t = 10+2ε ms, throttle is closed
}()
l := NewLogger(&buf)
t1 := make(chan *Update)
go func() {
- t1 <- &Update{"first", time.Now()} // t = 0+ε ms, throttle is open
- t1 <- &Update{"second", time.Now()} // t = 0+2ε ms, throttle is closed
- close(t1) // t = 0+3ε ms, throttle is closed
+ t1 <- &Update{"first", time.Now(), false} // t = 0+ε ms, throttle is open
+ t1 <- &Update{"second", time.Now(), false} // t = 0+2ε ms, throttle is closed
+ close(t1) // t = 0+3ε ms, throttle is closed
}()
l.enqueue(UnthrottledChanTask(t1))
import (
"fmt"
+ "math"
"sync/atomic"
"time"
)
u := &Update{
S: fmt.Sprintf("%s: %3.f%% (%d/%d)",
- c.msg, percentage, new, c.total),
+ c.msg, math.Floor(percentage), new, c.total),
At: time.Now(),
}
return new
}
+// Entry logs a line-delimited task entry.
+func (t *PercentageTask) Entry(update string) {
+ t.ch <- &Update{
+ S: fmt.Sprintf("%s\n", update),
+ At: time.Now(),
+ Force: true,
+ }
+}
+
// Updates implements Task.Updates and returns a channel which is written to
// when the state of this task changes, and closed when the task is completed.
// has been completed.
S string
// At is the time that this update was sent.
At time.Time
+
+ // Force determines if this update should not be throttled.
+ Force bool
+}
+
+// Throttled determines whether this update should be throttled, based on the
+// given earliest time of the next update. The caller should determine how often
+// updates should be throttled. An Update with Force=true is never throttled.
+func (u *Update) Throttled(next time.Time) bool {
+ return !(u.Force || u.At.After(next))
}
import (
"encoding/hex"
"fmt"
+ "strings"
"github.com/git-lfs/git-lfs/errors"
"github.com/git-lfs/git-lfs/git"
"github.com/git-lfs/git-lfs/git/githistory/log"
+ "github.com/git-lfs/git-lfs/tools"
)
// refUpdater is a type responsible for moving references from one point in the
list := r.Logger.List("migrate: Updating refs")
defer list.Complete()
+ var maxNameLen int
+ for _, ref := range r.Refs {
+ maxNameLen = tools.MaxInt(maxNameLen, len(ref.Name))
+ }
+
for _, ref := range r.Refs {
sha1, err := hex.DecodeString(ref.Sha)
if err != nil {
return err
}
- list.Entry(fmt.Sprintf(" %s\t%s -> %x", ref.Name, ref.Sha, to))
+ namePadding := tools.MaxInt(maxNameLen-len(ref.Name), 0)
+ list.Entry(fmt.Sprintf(" %s%s\t%s -> %x", ref.Name, strings.Repeat(" ", namePadding), ref.Sha, to))
}
return nil
// moved, and a reflog entry will be created.
UpdateRefs bool
+ // Verbose mode prints migrated objects.
+ Verbose bool
+
// BlobFn specifies a function to rewrite blobs.
//
// It is called once per unique, unchanged path. That is to say, if
return nil, err
}
- var p *log.PercentageTask
+ var perc *log.PercentageTask
if opt.UpdateRefs {
- p = r.l.Percentage("migrate: Rewriting commits", uint64(len(commits)))
+ perc = r.l.Percentage("migrate: Rewriting commits", uint64(len(commits)))
} else {
- p = r.l.Percentage("migrate: Examining commits", uint64(len(commits)))
+ perc = r.l.Percentage("migrate: Examining commits", uint64(len(commits)))
+ }
+
+ var vPerc *log.PercentageTask
+ if opt.Verbose {
+ vPerc = perc
}
// Keep track of the last commit that we rewrote. Callers often want
}
// Rewrite the tree given at that commit.
- rewrittenTree, err := r.rewriteTree(original.TreeID, "", opt.blobFn(), opt.treeFn())
+ rewrittenTree, err := r.rewriteTree(oid, original.TreeID, "", opt.blobFn(), opt.treeFn(), vPerc)
if err != nil {
return nil, err
}
r.cacheCommit(oid, newSha)
// Increment the percentage displayed in the terminal.
- p.Count(1)
+ perc.Count(1)
// Move the tip forward.
tip = newSha
}
}
- r.l.Close()
-
return tip, err
}
//
// It returns the new SHA of the rewritten tree, or an error if the tree was
// unable to be rewritten.
-func (r *Rewriter) rewriteTree(sha []byte, path string, fn BlobRewriteFn, tfn TreeCallbackFn) ([]byte, error) {
- tree, err := r.db.Tree(sha)
+func (r *Rewriter) rewriteTree(commitOID []byte, treeOID []byte, path string, fn BlobRewriteFn, tfn TreeCallbackFn, perc *log.PercentageTask) ([]byte, error) {
+ tree, err := r.db.Tree(treeOID)
if err != nil {
return nil, err
}
switch entry.Type() {
case odb.BlobObjectType:
- oid, err = r.rewriteBlob(entry.Oid, path, fn)
+ oid, err = r.rewriteBlob(commitOID, entry.Oid, path, fn, perc)
case odb.TreeObjectType:
- oid, err = r.rewriteTree(entry.Oid, path, fn, tfn)
+ oid, err = r.rewriteTree(commitOID, entry.Oid, path, fn, tfn, perc)
default:
oid = entry.Oid
}
if tree.Equal(rewritten) {
- return sha, nil
+ return treeOID, nil
}
return r.db.WriteTree(rewritten)
}
// database by the SHA1 "from" []byte. It writes and returns the new blob SHA,
// or an error if either the BlobRewriteFn returned one, or if the object could
// not be loaded/saved.
-func (r *Rewriter) rewriteBlob(from []byte, path string, fn BlobRewriteFn) ([]byte, error) {
+func (r *Rewriter) rewriteBlob(commitOID, from []byte, path string, fn BlobRewriteFn, perc *log.PercentageTask) ([]byte, error) {
blob, err := r.db.Blob(from)
if err != nil {
return nil, err
return nil, err
}
+ if perc != nil {
+ perc.Entry(fmt.Sprintf("migrate: commit %s: %s", hex.EncodeToString(commitOID), path))
+ }
+
return sha, nil
}
// delta-base chain successively to itself.
//
// If there was an error in the delta-base resolution, i.e., the chain
- // is malformed, has a bad instruction, or there was an mmap-related
- // read error, this function is expected to return that error.
+ // is malformed, has a bad instruction, or there was a file read error, this
+ // function is expected to return that error.
//
// In the event that a non-nil error is returned, it is assumed that the
// unpacked data this function returns is malformed, or otherwise
return int(i.fanout[255])
}
+// Close closes the packfile index if the underlying data stream is closeable.
+// If so, it returns any error involved in closing.
+func (i *Index) Close() error {
+ if close, ok := i.r.(io.Closer); ok {
+ return close.Close()
+ }
+ return nil
+}
+
var (
// errNotFound is an error returned by Index.Entry() (see: below) when
// an object cannot be found in the index.
// Close closes the packfile if the underlying data stream is closeable. If so,
// it returns any error involved in closing.
func (p *Packfile) Close() error {
+ var iErr error
+ if p.idx != nil {
+ iErr = p.idx.Close()
+ }
+
if close, ok := p.r.(io.Closer); ok {
return close.Close()
}
- return nil
+ return iErr
}
// Object returns a reference to an object packed in the receiving *Packfile. It
import (
"fmt"
+ "os"
"path/filepath"
"regexp"
"sort"
-
- "golang.org/x/exp/mmap"
)
// Set allows access of objects stored across a set of packfiles.
name := submatch[1]
- packf, err := mmap.Open(filepath.Join(pd, fmt.Sprintf("pack-%s.pack", name)))
+ packf, err := os.Open(filepath.Join(pd, fmt.Sprintf("pack-%s.pack", name)))
if err != nil {
return nil, err
}
- idxf, err := mmap.Open(filepath.Join(pd, fmt.Sprintf("pack-%s.idx", name)))
+ idxf, err := os.Open(filepath.Join(pd, fmt.Sprintf("pack-%s.idx", name)))
if err != nil {
return nil, err
}
-hash: f5514f65f17e413de9f48c316520c5b7f99003befccd7b89e48bd458438ab969
-updated: 2017-08-10T17:19:03.014970877-06:00
+hash: 7ac0f055e06001b80cc9cfc40de07c103b3b59abec0f3be0fca7cce49b71eec7
+updated: 2017-09-19T15:38:52.833333079-06:00
imports:
- name: github.com/bgentry/go-netrc
version: 9fd32a8b3d3d3f9d43c341bfe098430e07609480
- ntlm/md4
- name: github.com/xeipuuv/gojsonschema
version: 6b67b3fab74d992bd07f72550006ab2c6907c416
-- name: golang.org/x/exp
- version: be79676510e5ce293c77bb087cdab4796a461bdb
- subpackages:
- - mmap
testImports:
- name: github.com/davecgh/go-spew
version: 5215b55f46b2b919f50a1df0eaa5886afe4e3b3d
subpackages:
- difflib
- name: github.com/xeipuuv/gojsonpointer
- version: e0fe6f68307607d540ed8eac07a342c33fa1b54a
+ version: 6fe8760cad3569743d51ddbb243b26f8456742dc
- name: github.com/xeipuuv/gojsonreference
version: e02fc20de94c78484cd5ffb007f8af96be030a45
version: 6b67b3fab74d992bd07f72550006ab2c6907c416
- package: github.com/pkg/errors
version: c605e284fe17294bda444b34710735b29d1a9d90
-- package: golang.org/x/exp
- subpackages:
- - mmap
"net/http"
"net/url"
"os"
+ "regexp"
"strconv"
"strings"
"time"
"github.com/rubyist/tracerx"
)
-var UserAgent = "git-lfs"
-
const MediaType = "application/vnd.git-lfs+json; charset=utf-8"
+var (
+ UserAgent = "git-lfs"
+ httpRE = regexp.MustCompile(`\Ahttps?://`)
+)
+
func (c *Client) NewRequest(method string, e Endpoint, suffix string, body interface{}) (*http.Request, error) {
sshRes, err := c.SSH.Resolve(e, method)
if err != nil {
prefix = sshRes.Href
}
+ if !httpRE.MatchString(prefix) {
+ urlfragment := strings.SplitN(prefix, "?", 2)[0]
+ return nil, fmt.Errorf("missing protocol: %q", urlfragment)
+ }
+
req, err := http.NewRequest(method, joinURL(prefix, suffix), nil)
if err != nil {
return req, err
//
// See: https://git-scm.com/docs/gitcredentials#_requesting_credentials
// for more.
- AskPass string `os:"GIT_ASKPASS" git:"core.askpass"`
+ AskPass string `os:"GIT_ASKPASS" git:"core.askpass" os:"SSH_ASKPASS"`
// Cached is a boolean determining whether or not to enable the
// credential cacher.
Cached bool `git:"lfs.cachecredentials"`
// store it in the creds instance that we will return to the caller.
creds := make(Creds)
creds["username"] = strings.TrimSpace(user.String())
- creds["password"] = pass.String()
+ creds["password"] = strings.TrimSpace(pass.String())
return creds, nil
}
Name: git-lfs
-Version: 2.3.0
+Version: 2.3.1
Release: 1%{?dist}
Summary: Git extension for versioning large files
-"C:\Program Files (x86)\Inno Setup 5\iscc.exe" /Qp "windows-installer\git-lfs-installer.iss"
\ No newline at end of file
+"%ProgramFiles(x86)%\Inno Setup 5\iscc.exe" /Qp "%~dp0\windows-installer\inno-setup-git-lfs-installer.iss"
rm -rf vendor/github.com/ThomsonReutersEikon/go-ntlm/utils
rm -rf vendor/github.com/davecgh/go-spew
rm -rf vendor/github.com/pmezard/go-difflib
-
-# Remove all subpackages of golang.org/x/exp/... which aren't 'mmap'.
-find ./vendor/golang.org/x/exp/* -type d | grep -v mmap | xargs rm -rf
#define MyAppName "Git LFS"
+#define PathToX86Binary "..\..\git-lfs-x86.exe"
+#ifnexist PathToX86Binary
+ #pragma error PathToX86Binary + " does not exist, please build it first."
+#endif
+
+#define PathToX64Binary "..\..\git-lfs-x64.exe"
+#ifnexist PathToX64Binary
+ #pragma error PathToX64Binary + " does not exist, please build it first."
+#endif
+
; Arbitrarily choose the x86 executable here as both have the version embedded.
-#define MyVersionInfoVersion GetFileVersion("..\..\git-lfs-x86.exe")
+#define MyVersionInfoVersion GetFileVersion(PathToX86Binary)
; Misuse RemoveFileExt to strip the 4th patch-level version number.
#define MyAppVersion RemoveFileExt(MyVersionInfoVersion)
PrivilegesRequired=none
SetupIconFile=git-lfs-logo.ico
SolidCompression=yes
+UninstallDisplayIcon={app}\git-lfs.exe
UsePreviousAppDir=no
VersionInfoVersion={#MyVersionInfoVersion}
WizardImageFile=git-lfs-wizard-image.bmp
Filename: "{code:GetExistingGitInstallation}\git-lfs-uninstaller.exe"; Parameters: "/S"; Flags: skipifdoesntexist
[Files]
-Source: ..\..\git-lfs-x64.exe; DestDir: "{app}"; Flags: ignoreversion; DestName: "git-lfs.exe"; AfterInstall: InstallGitLFS; Check: Is64BitInstallMode
-Source: ..\..\git-lfs-x86.exe; DestDir: "{app}"; Flags: ignoreversion; DestName: "git-lfs.exe"; AfterInstall: InstallGitLFS; Check: not Is64BitInstallMode
+Source: {#PathToX86Binary}; DestDir: "{app}"; Flags: ignoreversion; DestName: "git-lfs.exe"; AfterInstall: InstallGitLFS; Check: not Is64BitInstallMode
+Source: {#PathToX64Binary}; DestDir: "{app}"; Flags: ignoreversion; DestName: "git-lfs.exe"; AfterInstall: InstallGitLFS; Check: Is64BitInstallMode
[Registry]
Root: HKLM; Subkey: "SYSTEM\CurrentControlSet\Control\Session Manager\Environment"; ValueType: expandsz; ValueName: "Path"; ValueData: "{olddata};{app}"; Check: IsAdminLoggedOn and NeedsAddPath('{app}')
}
}
- fmt.Print(answer)
+ fmt.Println(answer)
}
(
set -e
- reponame="askpass-with-environ"
+ reponame="askpass-with-git-environ"
setup_remote_repo "$reponame"
clone_repo "$reponame" "$reponame"
# $password is defined from test/cmd/lfstest-gitserver.go (see: skipIfBadAuth)
export LFS_ASKPASS_USERNAME="user"
export LFS_ASKPASS_PASSWORD="pass"
- GIT_ASKPASS="lfs-askpass" GIT_TRACE=1 GIT_CURL_VERBOSE=1 git push 2>&1 | tee push.log
+ GIT_ASKPASS="lfs-askpass" SSH_ASKPASS="dont-call-me" GIT_TRACE=1 GIT_CURL_VERBOSE=1 git push origin master 2>&1 | tee push.log
GITSERVER_USER="$(printf $GITSERVER | sed -e 's/http:\/\//http:\/\/user@/')"
grep "filling with GIT_ASKPASS: lfs-askpass Username for \"$GITSERVER/$reponame\"" push.log
grep "filling with GIT_ASKPASS: lfs-askpass Password for \"$GITSERVER_USER/$reponame\"" push.log
+ grep "master -> master" push.log
)
end_test
export LFS_ASKPASS_PASSWORD="pass"
git config "core.askPass" "lfs-askpass"
cat .git/config
- GIT_TRACE=1 GIT_CURL_VERBOSE=1 git push 2>&1 | tee push.log
+ SSH_ASKPASS="dont-call-me" GIT_TRACE=1 GIT_CURL_VERBOSE=1 git push origin master 2>&1 | tee push.log
GITSERVER_USER="$(printf $GITSERVER | sed -e 's/http:\/\//http:\/\/user@/')"
grep "filling with GIT_ASKPASS: lfs-askpass Username for \"$GITSERVER/$reponame\"" push.log
grep "filling with GIT_ASKPASS: lfs-askpass Password for \"$GITSERVER_USER/$reponame\"" push.log
+ grep "master -> master" push.log
+)
+end_test
+
+begin_test "askpass: push with SSH_ASKPASS"
+(
+ set -e
+
+ if [ ! -z "$TRAVIS" ] ; then
+ # This test is known to be broken on Travis, so we skip it if the $TRAVIS
+ # environment variable is set.
+ #
+ # See: https://github.com/git-lfs/git-lfs/pull/2500 for more.
+ exit 0
+ fi
+
+ reponame="askpass-with-ssh-environ"
+ setup_remote_repo "$reponame"
+ clone_repo "$reponame" "$reponame"
+
+ git lfs track "*.dat"
+ echo "hello" > a.dat
+
+ git add .gitattributes a.dat
+ git commit -m "initial commit"
+
+ # $password is defined from test/cmd/lfstest-gitserver.go (see: skipIfBadAuth)
+ export LFS_ASKPASS_USERNAME="user"
+ export LFS_ASKPASS_PASSWORD="pass"
+ SSH_ASKPASS="lfs-askpass" GIT_TRACE=1 GIT_CURL_VERBOSE=1 git push origin master 2>&1 | tee push.log
+
+ GITSERVER_USER="$(printf $GITSERVER | sed -e 's/http:\/\//http:\/\/user@/')"
+
+ grep "filling with GIT_ASKPASS: lfs-askpass Username for \"$GITSERVER/$reponame\"" push.log
+ grep "filling with GIT_ASKPASS: lfs-askpass Password for \"$GITSERVER_USER/$reponame\"" push.log
+ grep "master -> master" push.log
)
end_test
popd
)
end_test
+
+begin_test "clone empty repository"
+(
+ set -e
+
+ reponame="clone_empty"
+ setup_remote_repo "$reponame"
+
+ cd "$TRASHDIR"
+ git lfs clone "$GITSERVER/$reponame" "$reponame" 2>&1 | tee clone.log
+ if [ "0" -ne "${PIPESTATUS[0]}" ]; then
+ echo >&2 "fatal: expected clone to succeed ..."
+ exit 1
+ fi
+)
+end_test
+
+begin_test "clone bare empty repository"
+(
+ set -e
+
+ reponame="clone_bare_empty"
+ setup_remote_repo "$reponame"
+
+ cd "$TRASHDIR"
+ git lfs clone "$GITSERVER/$reponame" "$reponame" --bare 2>&1 | tee clone.log
+ if [ "0" -ne "${PIPESTATUS[0]}" ]; then
+ echo >&2 "fatal: expected clone to succeed ..."
+ exit 1
+ fi
+)
+end_test
\ No newline at end of file
[ "$(echo "$objectlist" | wc -l)" -eq 12 ]
)
end_test
+
+begin_test "custom-transfer-standalone-urlmatch"
+(
+ set -e
+
+ # setup a git repo to be used as a local repo, not remote
+ reponame="test-custom-transfer-standalone-urlmatch"
+ setup_remote_repo "$reponame"
+
+ # clone directly, not through lfstest-gitserver
+ clone_repo_url "$REMOTEDIR/$reponame.git" $reponame
+
+ # set up custom transfer adapter to use a specific transfer agent, using a URL prefix match
+ git config lfs.customtransfer.testcustom.path lfstest-standalonecustomadapter
+ git config lfs.customtransfer.testcustom.concurrent false
+ git config remote.origin.lfsurl https://git.example.com/example/path/to/repo
+ git config lfs.https://git.example.com/example/path/.standalonetransferagent testcustom
+ git config lfs.standalonetransferagent invalid-agent
+
+ # git config lfs.standalonetransferagent testcustom
+ export TEST_STANDALONE_BACKUP_PATH="$(pwd)/test-custom-transfer-standalone-urlmatch-backup"
+ mkdir -p $TEST_STANDALONE_BACKUP_PATH
+ rm -rf $TEST_STANDALONE_BACKUP_PATH/*
+
+ git lfs track "*.dat" 2>&1 | tee track.log
+ grep "Tracking \"\*.dat\"" track.log
+ git add .gitattributes
+ git commit -m "Tracking"
+
+ # set up a decent amount of data so that there's work for multiple concurrent adapters
+ echo "[
+ {
+ \"CommitDate\":\"$(get_date -10d)\",
+ \"Files\":[
+ {\"Filename\":\"verify.dat\",\"Size\":18,\"Data\":\"send-verify-action\"},
+ {\"Filename\":\"file1.dat\",\"Size\":1024},
+ {\"Filename\":\"file2.dat\",\"Size\":750}]
+ },
+ {
+ \"CommitDate\":\"$(get_date -7d)\",
+ \"Files\":[
+ {\"Filename\":\"file1.dat\",\"Size\":1050},
+ {\"Filename\":\"file3.dat\",\"Size\":660},
+ {\"Filename\":\"file4.dat\",\"Size\":230}]
+ },
+ {
+ \"CommitDate\":\"$(get_date -5d)\",
+ \"Files\":[
+ {\"Filename\":\"file5.dat\",\"Size\":1200},
+ {\"Filename\":\"file6.dat\",\"Size\":300}]
+ },
+ {
+ \"CommitDate\":\"$(get_date -2d)\",
+ \"Files\":[
+ {\"Filename\":\"file3.dat\",\"Size\":120},
+ {\"Filename\":\"file5.dat\",\"Size\":450},
+ {\"Filename\":\"file7.dat\",\"Size\":520},
+ {\"Filename\":\"file8.dat\",\"Size\":2048}]
+ }
+ ]" | lfstest-testutils addcommits
+
+ GIT_TRACE=1 GIT_TRANSFER_TRACE=1 git push origin master 2>&1 | tee pushcustom.log
+ # use PIPESTATUS otherwise we get exit code from tee
+ [ ${PIPESTATUS[0]} = "0" ]
+
+ # Make sure the lock verification is not attempted.
+ grep "locks/verify$" pushcustom.log && false
+
+ grep "xfer: started custom adapter process" pushcustom.log
+ grep "xfer\[lfstest-standalonecustomadapter\]:" pushcustom.log
+ grep "12 of 12 files" pushcustom.log
+
+ rm -rf .git/lfs/objects
+ GIT_TRACE=1 GIT_TRANSFER_TRACE=1 git lfs fetch --all 2>&1 | tee fetchcustom.log
+ [ ${PIPESTATUS[0]} = "0" ]
+
+ grep "xfer: started custom adapter process" fetchcustom.log
+ grep "xfer\[lfstest-standalonecustomadapter\]:" fetchcustom.log
+ grep "12 of 12 files" fetchcustom.log
+
+ grep "Terminating test custom adapter gracefully" fetchcustom.log
+
+ objectlist=`find .git/lfs/objects -type f`
+ [ "$(echo "$objectlist" | wc -l)" -eq 12 ]
+)
+end_test
--- /dev/null
+#!/usr/bin/env bash
+
+. "test/testlib.sh"
+
+reponame="$(basename "$0" ".sh")"
+
+begin_test "GIT_LFS_PROGRESS"
+(
+ set -e
+ setup_remote_repo "$reponame"
+ clone_repo "$reponame" repo
+
+ git lfs track "*.dat"
+ echo "a" > a.dat
+ echo "b" > b.dat
+ echo "c" > c.dat
+ echo "d" > d.dat
+ echo "e" > e.dat
+ git add .gitattributes *.dat
+ git commit -m "add files"
+ git push origin master 2>&1 | tee push.log
+ grep "(5 of 5 files)" push.log
+
+ cd ..
+ GIT_LFS_PROGRESS="$TRASHDIR/progress.log" git lfs clone "$GITSERVER/$reponame" clone
+ cat progress.log
+ grep "download 1/5" progress.log
+ grep "download 2/5" progress.log
+ grep "download 3/5" progress.log
+ grep "download 4/5" progress.log
+ grep "download 5/5" progress.log
+
+ GIT_LFS_SKIP_SMUDGE=1 git clone "$GITSERVER/$reponame" clone2
+ cd clone2
+
+ rm -rf "$TRASHDIR/progress.log" .git/lfs/objects
+ GIT_LFS_PROGRESS="$TRASHDIR/progress.log" git lfs fetch --all
+ cat ../progress.log
+ grep "download 1/5" ../progress.log
+ grep "download 2/5" ../progress.log
+ grep "download 3/5" ../progress.log
+ grep "download 4/5" ../progress.log
+ grep "download 5/5" ../progress.log
+
+ rm -rf "$TRASHDIR/progress.log"
+ GIT_LFS_PROGRESS="$TRASHDIR/progress.log" git lfs checkout
+ cat ../progress.log
+ grep "checkout 1/5" ../progress.log
+ grep "checkout 2/5" ../progress.log
+ grep "checkout 3/5" ../progress.log
+ grep "checkout 4/5" ../progress.log
+ grep "checkout 5/5" ../progress.log
+)
+end_test
import (
"fmt"
"net/http"
+ "regexp"
+ "strings"
"sync"
"github.com/git-lfs/git-lfs/lfsapi"
a.workerWait.Done()
}
+var httpRE = regexp.MustCompile(`\Ahttps?://`)
+
func (a *adapterBase) newHTTPRequest(method string, rel *Action) (*http.Request, error) {
+ if !httpRE.MatchString(rel.Href) {
+ urlfragment := strings.SplitN(rel.Href, "?", 2)[0]
+ return nil, fmt.Errorf("missing protocol: %q", urlfragment)
+ }
+
req, err := http.NewRequest(method, rel.Href, nil)
if err != nil {
return nil, err
type customAdapterInitRequest struct {
Event string `json:"event"`
Operation string `json:"operation"`
+ Remote string `json:"remote"`
Concurrent bool `json:"concurrent"`
ConcurrentTransfers int `json:"concurrenttransfers"`
}
-func NewCustomAdapterInitRequest(op string, concurrent bool, concurrentTransfers int) *customAdapterInitRequest {
- return &customAdapterInitRequest{"init", op, concurrent, concurrentTransfers}
+func NewCustomAdapterInitRequest(
+ op string, remote string, concurrent bool, concurrentTransfers int,
+) *customAdapterInitRequest {
+ return &customAdapterInitRequest{"init", op, remote, concurrent, concurrentTransfers}
}
type customAdapterTransferRequest struct {
ctx := &customAdapterWorkerContext{workerNum, cmd, outp, bufio.NewReader(outp), inp, tracer}
// send initiate message
- initReq := NewCustomAdapterInitRequest(a.getOperationName(), a.concurrent, a.originalConcurrency)
+ initReq := NewCustomAdapterInitRequest(
+ a.getOperationName(), a.remote, a.concurrent, a.originalConcurrency,
+ )
resp, err := a.exchangeMessage(ctx, initReq)
if err != nil {
a.abortWorkerProcess(ctx)
import (
"sync"
+ "github.com/git-lfs/git-lfs/config"
"github.com/git-lfs/git-lfs/lfsapi"
"github.com/rubyist/tracerx"
)
return m.concurrentTransfers
}
+func (m *Manifest) IsStandaloneTransfer() bool {
+ return m.standaloneTransferAgent != ""
+}
+
func (m *Manifest) batchClient() *tqClient {
if r := m.MaxRetries(); r > 0 {
m.tqClient.MaxRetries = r
}
func NewManifestWithClient(apiClient *lfsapi.Client) *Manifest {
+ return NewManifestClientOperationRemote(apiClient, "", "")
+}
+
+func NewManifestClientOperationRemote(
+ apiClient *lfsapi.Client, operation, remote string,
+) *Manifest {
m := &Manifest{
apiClient: apiClient,
tqClient: &tqClient{Client: apiClient},
m.concurrentTransfers = v
}
m.basicTransfersOnly = git.Bool("lfs.basictransfersonly", false)
- m.standaloneTransferAgent, _ = git.Get("lfs.standalonetransferagent")
+ m.standaloneTransferAgent = findStandaloneTransfer(
+ apiClient, operation, remote,
+ )
tusAllowed = git.Bool("lfs.tustransfers", false)
configureCustomAdapters(git, m)
}
return m
}
+func findStandaloneTransfer(client *lfsapi.Client, operation, remote string) string {
+ if operation == "" || remote == "" {
+ v, _ := client.GitEnv().Get("lfs.standalonetransferagent")
+ return v
+ }
+
+ ep := client.Endpoints.RemoteEndpoint(operation, remote)
+ uc := config.NewURLConfig(client.GitEnv())
+ v, ok := uc.Get("lfs", ep.Url, "standalonetransferagent")
+ if !ok {
+ return ""
+ }
+
+ return v
+}
+
// GetAdapterNames returns a list of the names of adapters available to be created
func (m *Manifest) GetAdapterNames(dir Direction) []string {
switch dir {
outError error
}
-func NewJsonPointer(jsonPointerString string) (JsonPointer, error) {
-
- var p JsonPointer
- err := p.parse(jsonPointerString)
- return p, err
-
-}
-
type JsonPointer struct {
referenceTokens []string
}
-// "Constructor", parses the given string JSON pointer
-func (p *JsonPointer) parse(jsonPointerString string) error {
+// NewJsonPointer parses the given string JSON pointer and returns an object
+func NewJsonPointer(jsonPointerString string) (p JsonPointer, err error) {
- var err error
-
- if jsonPointerString != const_empty_pointer {
- if !strings.HasPrefix(jsonPointerString, const_pointer_separator) {
- err = errors.New(const_invalid_start)
- } else {
- referenceTokens := strings.Split(jsonPointerString, const_pointer_separator)
- for _, referenceToken := range referenceTokens[1:] {
- p.referenceTokens = append(p.referenceTokens, referenceToken)
- }
- }
+ // Pointer to the root of the document
+ if len(jsonPointerString) == 0 {
+ // Keep referenceTokens nil
+ return
+ }
+ if jsonPointerString[0] != '/' {
+ return p, errors.New(const_invalid_start)
}
- return err
+ p.referenceTokens = strings.Split(jsonPointerString[1:], const_pointer_separator)
+ return
}
// Uses the pointer to retrieve a value from a JSON document
for ti, token := range p.referenceTokens {
- decodedToken := decodeReferenceToken(token)
isLastToken := ti == len(p.referenceTokens)-1
- rValue := reflect.ValueOf(node)
- kind = rValue.Kind()
+ switch v := node.(type) {
- switch kind {
-
- case reflect.Map:
- m := node.(map[string]interface{})
- if _, ok := m[decodedToken]; ok {
- node = m[decodedToken]
+ case map[string]interface{}:
+ decodedToken := decodeReferenceToken(token)
+ if _, ok := v[decodedToken]; ok {
+ node = v[decodedToken]
if isLastToken && i.mode == "SET" {
- m[decodedToken] = i.setInValue
+ v[decodedToken] = i.setInValue
}
} else {
- i.outError = errors.New(fmt.Sprintf("Object has no key '%s'", token))
- i.getOutKind = kind
+ i.outError = fmt.Errorf("Object has no key '%s'", decodedToken)
+ i.getOutKind = reflect.Map
i.getOutNode = nil
return
}
- case reflect.Slice:
- s := node.([]interface{})
+ case []interface{}:
tokenIndex, err := strconv.Atoi(token)
if err != nil {
- i.outError = errors.New(fmt.Sprintf("Invalid array index '%s'", token))
- i.getOutKind = kind
+ i.outError = fmt.Errorf("Invalid array index '%s'", token)
+ i.getOutKind = reflect.Slice
i.getOutNode = nil
return
}
- sLength := len(s)
- if tokenIndex < 0 || tokenIndex >= sLength {
- i.outError = errors.New(fmt.Sprintf("Out of bound array[0,%d] index '%d'", sLength, tokenIndex))
- i.getOutKind = kind
+ if tokenIndex < 0 || tokenIndex >= len(v) {
+ i.outError = fmt.Errorf("Out of bound array[0,%d] index '%d'", len(v), tokenIndex)
+ i.getOutKind = reflect.Slice
i.getOutNode = nil
return
}
- node = s[tokenIndex]
+ node = v[tokenIndex]
if isLastToken && i.mode == "SET" {
- s[tokenIndex] = i.setInValue
+ v[tokenIndex] = i.setInValue
}
default:
- i.outError = errors.New(fmt.Sprintf("Invalid token reference '%s'", token))
- i.getOutKind = kind
+ i.outError = fmt.Errorf("Invalid token reference '%s'", token)
+ i.getOutKind = reflect.ValueOf(node).Kind()
i.getOutNode = nil
return
}
}
- rValue := reflect.ValueOf(node)
- kind = rValue.Kind()
-
i.getOutNode = node
- i.getOutKind = kind
+ i.getOutKind = reflect.ValueOf(node).Kind()
i.outError = nil
}
// ~1 => /
// ... and vice versa
-const (
- const_encoded_reference_token_0 = `~0`
- const_encoded_reference_token_1 = `~1`
- const_decoded_reference_token_0 = `~`
- const_decoded_reference_token_1 = `/`
-)
-
func decodeReferenceToken(token string) string {
- step1 := strings.Replace(token, const_encoded_reference_token_1, const_decoded_reference_token_1, -1)
- step2 := strings.Replace(step1, const_encoded_reference_token_0, const_decoded_reference_token_0, -1)
+ step1 := strings.Replace(token, `~1`, `/`, -1)
+ step2 := strings.Replace(step1, `~0`, `~`, -1)
return step2
}
func encodeReferenceToken(token string) string {
- step1 := strings.Replace(token, const_decoded_reference_token_1, const_encoded_reference_token_1, -1)
- step2 := strings.Replace(step1, const_decoded_reference_token_0, const_encoded_reference_token_0, -1)
+ step1 := strings.Replace(token, `~`, `~0`, -1)
+ step2 := strings.Replace(step1, `/`, `~1`, -1)
return step2
}
// author xeipuuv
// author-github https://github.com/xeipuuv
// author-mail xeipuuv@gmail.com
-//
+//
// repository-name gojsonpointer
// repository-desc An implementation of JSON Pointer - Go language
-//
+//
// description Automated tests on package.
-//
+//
// created 03-03-2013
package gojsonpointer
)
const (
- TEST_DOCUMENT_NB_ELEMENTS = 11
+ TEST_DOCUMENT_NB_ELEMENTS = 13
TEST_NODE_OBJ_NB_ELEMENTS = 4
TEST_DOCUMENT_STRING = `{
"foo": ["bar", "baz"],
"i\\j": 5,
"k\"l": 6,
" ": 7,
-"m~n": 8
+"m~n": 8,
+"o~/p": 9,
+"q/~r": 10
}`
)
func TestEscaping(t *testing.T) {
- ins := []string{`/`, `/`, `/a~1b`, `/a~1b`, `/c%d`, `/e^f`, `/g|h`, `/i\j`, `/k"l`, `/ `, `/m~0n`}
- outs := []float64{0, 0, 1, 1, 2, 3, 4, 5, 6, 7, 8}
+ ins := []string{`/`, `/`, `/a~1b`, `/a~1b`, `/c%d`, `/e^f`, `/g|h`, `/i\j`, `/k"l`, `/ `, `/m~0n`, `/o~0~1p`, `/q~1~0r`}
+ outs := []float64{0, 0, 1, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10}
for i := range ins {
if err != nil {
t.Errorf("NewJsonPointer(%v) error %v", ins[i], err.Error())
}
+ s := p.String()
+ if s != ins[i] {
+ t.Errorf("\"%v\" -> \"%v\"", ins[i], s)
+ }
result, _, err := p.Get(testDocumentJson)
if err != nil {
}
+func BenchmarkParse(b *testing.B) {
+ for i := 0; i < b.N; i++ {
+ NewJsonPointer(`/definitions/simple/0/next`)
+ }
+}
+
+func BenchmarkParseWithEscape(b *testing.B) {
+ for i := 0; i < b.N; i++ {
+ NewJsonPointer(`/definiti~0ons/simple/0/next`)
+ }
+}
+
+func BenchmarkString(b *testing.B) {
+ p, _ := NewJsonPointer(`/definitions/simple/0/next`)
+ b.ResetTimer()
+ for i := 0; i < b.N; i++ {
+ p.String()
+ }
+}
+
+func BenchmarkStringWithEscape(b *testing.B) {
+ p, _ := NewJsonPointer(`/definiti~0ons/simple/0/next`)
+ b.ResetTimer()
+ for i := 0; i < b.N; i++ {
+ p.String()
+ }
+}
+
func TestFullDocument(t *testing.T) {
in := ``
}
}
+func BenchmarkGet(b *testing.B) {
+ p, _ := NewJsonPointer(`/obj/d/1/f`)
+ b.ResetTimer()
+ for i := 0; i < b.N; i++ {
+ p.Get(testDocumentJson)
+ }
+}
+
func TestArray(t *testing.T) {
ins := []string{`/foo/0`, `/foo/0`, `/foo/1`}
+++ /dev/null
-# Treat all files in this repo as binary, with no git magic updating
-# line endings. Windows users contributing to Go will need to use a
-# modern version of git and editors capable of LF line endings.
-#
-# We'll prevent accidental CRLF line endings from entering the repo
-# via the git-review gofmt checks.
-#
-# See golang.org/issue/9281
-
-* -text
+++ /dev/null
-# Add no patterns to .hgignore except for files generated by the build.
-last-change
+++ /dev/null
-# This source code refers to The Go Authors for copyright purposes.
-# The master list of authors is in the main Go distribution,
-# visible at http://tip.golang.org/AUTHORS.
+++ /dev/null
-# Contributing to Go
-
-Go is an open source project.
-
-It is the work of hundreds of contributors. We appreciate your help!
-
-
-## Filing issues
-
-When [filing an issue](https://golang.org/issue/new), make sure to answer these five questions:
-
-1. What version of Go are you using (`go version`)?
-2. What operating system and processor architecture are you using?
-3. What did you do?
-4. What did you expect to see?
-5. What did you see instead?
-
-General questions should go to the [golang-nuts mailing list](https://groups.google.com/group/golang-nuts) instead of the issue tracker.
-The gophers there will answer or ask you to file an issue if you've tripped over a bug.
-
-## Contributing code
-
-Please read the [Contribution Guidelines](https://golang.org/doc/contribute.html)
-before sending patches.
-
-**We do not accept GitHub pull requests**
-(we use [Gerrit](https://code.google.com/p/gerrit/) instead for code review).
-
-Unless otherwise noted, the Go source files are distributed under
-the BSD-style license found in the LICENSE file.
-
+++ /dev/null
-# This source code was written by the Go contributors.
-# The master list of contributors is in the main Go distribution,
-# visible at http://tip.golang.org/CONTRIBUTORS.
+++ /dev/null
-Copyright (c) 2009 The Go Authors. All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are
-met:
-
- * Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
- * Redistributions in binary form must reproduce the above
-copyright notice, this list of conditions and the following disclaimer
-in the documentation and/or other materials provided with the
-distribution.
- * Neither the name of Google Inc. nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+++ /dev/null
-Additional IP Rights Grant (Patents)
-
-"This implementation" means the copyrightable works distributed by
-Google as part of the Go project.
-
-Google hereby grants to You a perpetual, worldwide, non-exclusive,
-no-charge, royalty-free, irrevocable (except as stated in this section)
-patent license to make, have made, use, offer to sell, sell, import,
-transfer and otherwise run, modify and propagate the contents of this
-implementation of Go, where such license applies only to those patent
-claims, both currently owned or controlled by Google and acquired in
-the future, licensable by Google that are necessarily infringed by this
-implementation of Go. This grant does not include claims that would be
-infringed only as a consequence of further modification of this
-implementation. If you or your agent or exclusive licensee institute or
-order or agree to the institution of patent litigation against any
-entity (including a cross-claim or counterclaim in a lawsuit) alleging
-that this implementation of Go or any code incorporated within this
-implementation of Go constitutes direct or contributory patent
-infringement, or inducement of patent infringement, then any patent
-rights granted to you under this License for this implementation of Go
-shall terminate as of the date such litigation is filed.
+++ /dev/null
-This subrepository holds experimental and deprecated (in the "old"
-directory) packages.
-
-The idea for this subrepository originated as the "pkg/exp" directory
-of the main repository, but its presence there made it unavailable
-to users of the binary downloads of the Go installation. The
-subrepository has therefore been created to make it possible to "go
-get" these packages.
-
-Warning: Packages here are experimental and unreliable. Some may
-one day be promoted to the main repository or other subrepository,
-or they may be modified arbitrarily or even disappear altogether.
-
-In short, code in this subrepository is not subject to the Go 1
-compatibility promise. (No subrepo is, but the promise is even more
-likely to be violated by go.exp than the others.)
-
-Caveat emptor.
+++ /dev/null
-issuerepo: golang/go
+++ /dev/null
-// Copyright 2015 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build ignore
-//
-// This build tag means that "go build" does not build this file. Use "go run
-// manual_test_program.go" to run it.
-//
-// You will also need to change "debug = false" to "debug = true" in mmap_*.go.
-
-package main
-
-import (
- "log"
- "math/rand"
- "time"
-
- "golang.org/x/exp/mmap"
-)
-
-var garbage []byte
-
-func main() {
- const filename = "manual_test_program.go"
-
- for _, explicitClose := range []bool{false, true} {
- r, err := mmap.Open(filename)
- if err != nil {
- log.Fatalf("Open: %v", err)
- }
- if explicitClose {
- r.Close()
- } else {
- // Leak the *mmap.ReaderAt returned by mmap.Open. The finalizer
- // should pick it up, if finalizers run at all.
- }
- }
-
- println("Finished all explicit Close calls.")
- println("Creating and collecting garbage.")
- println("Look for two munmap log messages.")
- println("Hit Ctrl-C to exit.")
-
- rng := rand.New(rand.NewSource(1))
- now := time.Now()
- for {
- garbage = make([]byte, rng.Intn(1<<20))
- if time.Since(now) > 1*time.Second {
- now = time.Now()
- print(".")
- }
- }
-}
+++ /dev/null
-// Copyright 2015 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build !linux,!windows,!darwin
-
-// Package mmap provides a way to memory-map a file.
-package mmap
-
-import (
- "fmt"
- "os"
-)
-
-// ReaderAt reads a memory-mapped file.
-//
-// Like any io.ReaderAt, clients can execute parallel ReadAt calls, but it is
-// not safe to call Close and reading methods concurrently.
-type ReaderAt struct {
- f *os.File
- len int
-}
-
-// Close closes the reader.
-func (r *ReaderAt) Close() error {
- return r.f.Close()
-}
-
-// Len returns the length of the underlying memory-mapped file.
-func (r *ReaderAt) Len() int {
- return r.len
-}
-
-// At returns the byte at index i.
-func (r *ReaderAt) At(i int) byte {
- if i < 0 || r.len <= i {
- panic("index out of range")
- }
- var b [1]byte
- r.ReadAt(b[:], int64(i))
- return b[0]
-}
-
-// ReadAt implements the io.ReaderAt interface.
-func (r *ReaderAt) ReadAt(p []byte, off int64) (int, error) {
- return r.f.ReadAt(p, off)
-}
-
-// Open memory-maps the named file for reading.
-func Open(filename string) (*ReaderAt, error) {
- f, err := os.Open(filename)
- if err != nil {
- return nil, err
- }
- fi, err := f.Stat()
- if err != nil {
- f.Close()
- return nil, err
- }
-
- size := fi.Size()
- if size < 0 {
- f.Close()
- return nil, fmt.Errorf("mmap: file %q has negative size", filename)
- }
- if size != int64(int(size)) {
- f.Close()
- return nil, fmt.Errorf("mmap: file %q is too large", filename)
- }
-
- return &ReaderAt{
- f: f,
- len: int(fi.Size()),
- }, nil
-}
+++ /dev/null
-// Copyright 2015 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package mmap
-
-import (
- "bytes"
- "io"
- "io/ioutil"
- "testing"
-)
-
-func TestOpen(t *testing.T) {
- const filename = "mmap_test.go"
- r, err := Open(filename)
- if err != nil {
- t.Fatalf("Open: %v", err)
- }
- got := make([]byte, r.Len())
- if _, err := r.ReadAt(got, 0); err != nil && err != io.EOF {
- t.Fatalf("ReadAt: %v", err)
- }
- want, err := ioutil.ReadFile(filename)
- if err != nil {
- t.Fatalf("ioutil.ReadFile: %v", err)
- }
- if len(got) != len(want) {
- t.Fatalf("got %d bytes, want %d", len(got), len(want))
- }
- if !bytes.Equal(got, want) {
- t.Fatalf("\ngot %q\nwant %q", string(got), string(want))
- }
-}
+++ /dev/null
-// Copyright 2015 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build linux darwin
-
-// Package mmap provides a way to memory-map a file.
-package mmap
-
-import (
- "errors"
- "fmt"
- "io"
- "os"
- "runtime"
- "syscall"
-)
-
-// debug is whether to print debugging messages for manual testing.
-//
-// The runtime.SetFinalizer documentation says that, "The finalizer for x is
-// scheduled to run at some arbitrary time after x becomes unreachable. There
-// is no guarantee that finalizers will run before a program exits", so we
-// cannot automatically test that the finalizer runs. Instead, set this to true
-// when running the manual test.
-const debug = false
-
-// ReaderAt reads a memory-mapped file.
-//
-// Like any io.ReaderAt, clients can execute parallel ReadAt calls, but it is
-// not safe to call Close and reading methods concurrently.
-type ReaderAt struct {
- data []byte
-}
-
-// Close closes the reader.
-func (r *ReaderAt) Close() error {
- if r.data == nil {
- return nil
- }
- data := r.data
- r.data = nil
- if debug {
- var p *byte
- if len(data) != 0 {
- p = &data[0]
- }
- println("munmap", r, p)
- }
- runtime.SetFinalizer(r, nil)
- return syscall.Munmap(data)
-}
-
-// Len returns the length of the underlying memory-mapped file.
-func (r *ReaderAt) Len() int {
- return len(r.data)
-}
-
-// At returns the byte at index i.
-func (r *ReaderAt) At(i int) byte {
- return r.data[i]
-}
-
-// ReadAt implements the io.ReaderAt interface.
-func (r *ReaderAt) ReadAt(p []byte, off int64) (int, error) {
- if r.data == nil {
- return 0, errors.New("mmap: closed")
- }
- if off < 0 || int64(len(r.data)) < off {
- return 0, fmt.Errorf("mmap: invalid ReadAt offset %d", off)
- }
- n := copy(p, r.data[off:])
- if n < len(p) {
- return n, io.EOF
- }
- return n, nil
-}
-
-// Open memory-maps the named file for reading.
-func Open(filename string) (*ReaderAt, error) {
- f, err := os.Open(filename)
- if err != nil {
- return nil, err
- }
- defer f.Close()
- fi, err := f.Stat()
- if err != nil {
- return nil, err
- }
-
- size := fi.Size()
- if size == 0 {
- return &ReaderAt{}, nil
- }
- if size < 0 {
- return nil, fmt.Errorf("mmap: file %q has negative size", filename)
- }
- if size != int64(int(size)) {
- return nil, fmt.Errorf("mmap: file %q is too large", filename)
- }
-
- data, err := syscall.Mmap(int(f.Fd()), 0, int(size), syscall.PROT_READ, syscall.MAP_SHARED)
- if err != nil {
- return nil, err
- }
- r := &ReaderAt{data}
- if debug {
- var p *byte
- if len(data) != 0 {
- p = &data[0]
- }
- println("mmap", r, p)
- }
- runtime.SetFinalizer(r, (*ReaderAt).Close)
- return r, nil
-}
+++ /dev/null
-// Copyright 2015 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// Package mmap provides a way to memory-map a file.
-package mmap
-
-import (
- "errors"
- "fmt"
- "io"
- "os"
- "runtime"
- "syscall"
- "unsafe"
-)
-
-// debug is whether to print debugging messages for manual testing.
-//
-// The runtime.SetFinalizer documentation says that, "The finalizer for x is
-// scheduled to run at some arbitrary time after x becomes unreachable. There
-// is no guarantee that finalizers will run before a program exits", so we
-// cannot automatically test that the finalizer runs. Instead, set this to true
-// when running the manual test.
-const debug = false
-
-// ReaderAt reads a memory-mapped file.
-//
-// Like any io.ReaderAt, clients can execute parallel ReadAt calls, but it is
-// not safe to call Close and reading methods concurrently.
-type ReaderAt struct {
- data []byte
-}
-
-// Close closes the reader.
-func (r *ReaderAt) Close() error {
- if r.data == nil {
- return nil
- }
- data := r.data
- r.data = nil
- if debug {
- var p *byte
- if len(data) != 0 {
- p = &data[0]
- }
- println("munmap", r, p)
- }
- runtime.SetFinalizer(r, nil)
- return syscall.UnmapViewOfFile(uintptr(unsafe.Pointer(&data[0])))
-}
-
-// Len returns the length of the underlying memory-mapped file.
-func (r *ReaderAt) Len() int {
- return len(r.data)
-}
-
-// At returns the byte at index i.
-func (r *ReaderAt) At(i int) byte {
- return r.data[i]
-}
-
-// ReadAt implements the io.ReaderAt interface.
-func (r *ReaderAt) ReadAt(p []byte, off int64) (int, error) {
- if r.data == nil {
- return 0, errors.New("mmap: closed")
- }
- if off < 0 || int64(len(r.data)) < off {
- return 0, fmt.Errorf("mmap: invalid ReadAt offset %d", off)
- }
- n := copy(p, r.data[off:])
- if n < len(p) {
- return n, io.EOF
- }
- return n, nil
-}
-
-// Open memory-maps the named file for reading.
-func Open(filename string) (*ReaderAt, error) {
- f, err := os.Open(filename)
- if err != nil {
- return nil, err
- }
- defer f.Close()
- fi, err := f.Stat()
- if err != nil {
- return nil, err
- }
-
- size := fi.Size()
- if size == 0 {
- return &ReaderAt{}, nil
- }
- if size < 0 {
- return nil, fmt.Errorf("mmap: file %q has negative size", filename)
- }
- if size != int64(int(size)) {
- return nil, fmt.Errorf("mmap: file %q is too large", filename)
- }
-
- low, high := uint32(size), uint32(size>>32)
- fmap, err := syscall.CreateFileMapping(syscall.Handle(f.Fd()), nil, syscall.PAGE_READONLY, high, low, nil)
- if err != nil {
- return nil, err
- }
- defer syscall.CloseHandle(fmap)
- ptr, err := syscall.MapViewOfFile(fmap, syscall.FILE_MAP_READ, 0, 0, uintptr(size))
- if err != nil {
- return nil, err
- }
- data := (*[1 << 30]byte)(unsafe.Pointer(ptr))[:size]
-
- r := &ReaderAt{data: data}
- if debug {
- var p *byte
- if len(data) != 0 {
- p = &data[0]
- }
- println("mmap", r, p)
- }
- runtime.SetFinalizer(r, (*ReaderAt).Close)
- return r, nil
-}
"FileVersion": {
"Major": 2,
"Minor": 3,
- "Patch": 0,
+ "Patch": 1,
"Build": 0
}
},
projects / scm / test.git / commitdiff