Merge pull request JuliaLang#5528 from ivarne/distributing

make source-dist + git version info separation

.gitignore

@@ -6,6 +6,8 @@
 /usr
 /Make.user
 /julia-*
+/source-dist.tmp
+/source-dist.tmp1
 
 *.exe
 *.dll

DISTRIBUTING.md

@@ -16,6 +16,26 @@ GPL licensed, as various dependent libraries such as `FFTW`, `Rmath`,
 `SuiteSparse`, and `git` are GPL licensed. We do hope to have a
 non-GPL distribution of Julia in the future.
 
+Versioning and Git
+------------------
+The Makefile uses both the `VERSION` file and commit hashes and tags from the
+git repository to generate the `base/version_git.jl` with information we use to
+fill the splash screen and the `versioninfo()` output. If you for some reason
+don't want to have the git repository available when building you should
+pregenerate the `base/version_git.jl` file with:
+
+ make -C base version_git.jl.phony
+
+Juila has lots of build dependencies where we use patched versions that has not
+yet been included by the popular package managers. These dependencies will usually
+be automatically downloaded when you build, but if you want to be able to build
+Julia on a computer without internet access you should create a source-dist archive
+with the special make targed
+
+ make source-dist
+
+that creates a julia-version-commit.tar.gz archive with all required dependencies.
+
 When compiling a tagged release in the git repository, we don't display the
 branch/commit hash info in the splash screen. You can use this line to show
 a release description of up to 45 characters. To set this line you have
@@ -36,16 +56,6 @@ for Debian and Ubuntu-based systems. Although we have not yet experimented
 with it, [Alien](https://wiki.debian.org/Alien) could be used to
 generate Julia packages for various Linux distributions.
 
-Julia looks for git versioning information when building. If it does
-not find the git executable or the `.git/` directory is unreadable,
-the build process will continue, however all versioning information
-will be unavailable. This is the case, for instance, on Canonical's
-build servers where the [Ubuntu
-nightlies](https://launchpad.net/~staticfloat/+archive/julianightlies)
-are built. Therefore, a workaround must be enacted, where the git
-versioning information [is encoded into the
-source](https://github.com/staticfloat/julia-nightly-packaging/blob/master/build_ubuntu.sh#L76-78) before upload for building, and the source is modified to [not attempt to look for it](https://github.com/staticfloat/julia-nightly-packaging/blob/master/nogit-workaround.patch) when building.
-
 By default, Julia loads `$PREFIX/etc/julia/juliarc.jl` as an
 installation-wide initialization file. This file can be used by
 distribution managers to provide paths to various binaries such as a

Make.inc

@@ -50,8 +50,19 @@ endif
 # disable automatic Makefile rules
 .SUFFIXES:
 
+# find out if git repository is availible
+ifeq ($(shell [ -e $(JULIAHOME)/.git ] && echo true || echo "Warning: git information unavailable; versioning information limited" >&2), true)
+NO_GIT = 0
+else
+NO_GIT = 1
+endif
+
 JULIA_VERSION = $(shell cat $(JULIAHOME)/VERSION)
+ifneq ($(NO_GIT), 1)
 JULIA_COMMIT = $(shell git rev-parse --short=10 HEAD)
+else
+JULIA_COMMIT = $JULIA_VERSION
+endif
 
 # LLVM Options
 LLVMROOT = $(BUILD)

Makefile

@@ -32,6 +32,13 @@ endif
 $(foreach dir,$(DIRS),$(eval $(call dir_target,$(dir))))
 $(foreach link,base test doc examples,$(eval $(call symlink_target,$(link),$(BUILD)/share/julia)))
 
+git-submodules:
+ifneq ($(NO_GIT), 1)
+ @-git submodule update --init
+else
+ $(warn "Submodules could not be updated because git is unavailible")
+endif
+
 debug release: | $(DIRS) $(BUILD)/share/julia/base $(BUILD)/share/julia/test $(BUILD)/share/julia/doc $(BUILD)/share/julia/examples $(BUILD)/etc/julia/juliarc.jl
  @$(MAKE) $(QUIET_MAKE) julia-$@
  @export JL_PRIVATE_LIBDIR=$(JL_PRIVATE_LIBDIR) && \
@@ -43,9 +50,7 @@ julia-debug-symlink:
 julia-release-symlink:
  @ln -sf $(BUILD)/bin/julia-$(DEFAULT_REPL) julia
 
-julia-debug julia-release:
- @-git submodule init --quiet
- @-git submodule update
+julia-debug julia-release: git-submodules
  @$(MAKE) $(QUIET_MAKE) -C deps
  @$(MAKE) $(QUIET_MAKE) -C src lib$@
  @$(MAKE) $(QUIET_MAKE) -C base
@@ -261,6 +266,27 @@ else
 endif
  rm -fr julia-$(JULIA_COMMIT)
 
+
+source-dist: git-submodules
+ # Save git information
+ -@$(MAKE) -C base version_git.jl.phony
+ # Get all the dependencies downloaded
+ @$(MAKE) -C deps getall
+
+ # Create file source-dist.tmp to hold all the filenames that goes into the tarball
+ echo "base/version_git.jl" > source-dist.tmp
+ git ls-files >> source-dist.tmp
+ ls deps/*.tar.gz deps/*.tar.bz2 deps/*.tgz deps/random/*.tar.gz >> source-dist.tmp
+ git submodule --quiet foreach 'git ls-files | sed "s&^&$$path/&"' >> source-dist.tmp
+
+ # Remove unwanted files
+ sed '/\.git/d' source-dist.tmp > source-dist.tmp1
+ sed '/\.travis/d' source-dist.tmp1 > source-dist.tmp
+
+ # Create tarball
+ tar -cz -T source-dist.tmp --no-recursion -f julia-$(JULIA_VERSION)_$(JULIA_COMMIT).tar.gz
+ rm -f source-dist.tmp source-dist.tmp1
+
 clean: | $(CLEAN_TARGETS)
  @$(MAKE) -C base clean
  @$(MAKE) -C src clean
@@ -272,6 +298,7 @@ clean: | $(CLEAN_TARGETS)
  @rm -f julia
  @rm -f *~ *# *.tar.gz
  @rm -fr $(BUILD)/$(JL_PRIVATE_LIBDIR)
+ @rm -f source-dist.tmp source-dist.tmp1
 # Temporarily add this line to the Makefile to remove extras
  @rm -fr $(BUILD)/share/julia/extras
 
@@ -291,7 +318,7 @@ distclean: cleanall
 .PHONY: default debug release julia-debug julia-release \
  test testall testall1 test-* clean distclean cleanall \
  run-julia run-julia-debug run-julia-release run \
- install dist
+ install dist source-dist git-submodules
 
 ifeq ($(VERBOSE),1)
 .SILENT:

base/.gitignore

@@ -4,3 +4,4 @@
 /fenv_constants.jl
 /file_constants.jl
 /uv_constants.jl
+/version_git.jl

base/Makefile

@@ -3,38 +3,9 @@ include ../Make.inc
 
 PCRE_CONST = 0x[0-9a-fA-F]+|[-+]?\s*[0-9]+
 
-# These are all the values needed for the BUILD_INFO struct in build_h.jl
-version_string = $(shell cat ../VERSION)
-commit = $(shell git rev-parse HEAD 2>/dev/null)
-commit_short = $(shell git rev-parse --short HEAD 2>/dev/null)
-git_branch = $(shell git branch 2>/dev/null | sed -n '/\* /s///p')
-
-last_tag = $(shell git describe --tags --abbrev=0 2>/dev/null)
-tagged_commit = $(shell [ $$(git describe --tags --exact-match 2>/dev/null) ] && echo true || echo false)
-
-origin = $(shell [ -d ../.git/refs/remotes/origin ] && echo "origin/")
-
-build_number = $(shell git rev-list --count HEAD ^"$(last_tag)" 2>/dev/null || echo 0)
-fork_master_distance = $(shell git rev-list --count HEAD ^"$(origin)master" 2>/dev/null || echo 0)
-fork_master_timestamp = $(shell git show -s $$(git merge-base HEAD $(origin)master 2>/dev/null) --format=format:"%ct" 2>/dev/null || echo 0)
-
-git_time = $(shell git log -1 --pretty=format:%ct 2>/dev/null)
-ifneq ($(git_time), )
- ifneq (,$(filter $(OS), Darwin FreeBSD))
- date_string = "$(shell /bin/date -jr $(git_time) -u '+%Y-%m-%d %H:%M %Z')"
- else
- date_string = "$(shell /bin/date --date=@$(git_time) -u '+%Y-%m-%d %H:%M %Z')"
- endif
-else
- date_string = ""
-endif
-
-dirty = $(shell [ -z "$(shell git status --porcelain 2>/dev/null)" ] && echo "" || echo "*" )
-
 TAGGED_RELEASE_BANNER = ""
 
-
-all: pcre_h.jl errno_h.jl build_h.jl.phony fenv_constants.jl file_constants.jl uv_constants.jl
+all: pcre_h.jl errno_h.jl build_h.jl.phony fenv_constants.jl file_constants.jl uv_constants.jl version_git.jl.phony
 
 pcre_h.jl:
  @$(call PRINT_PERL, $(CPP) -dM $(shell $(PCRE_CONFIG) --prefix)/include/pcre.h | perl -nle '/^\s*#define\s+PCRE_(\w*)\s*\(?($(PCRE_CONST))\)?\s*$$/ and print "const $$1 = uint32($$2)"' | sort > $@)
@@ -69,33 +40,8 @@ else
  @echo "const USE_BLAS64 = false" >> $@
 endif
  @echo "const SYSCONFDIR = \"$(SYSCONFDIR)\"" >> $@
-
- @echo "immutable BuildInfo" >> $@
- @echo " version_string::String" >> $@
- @echo " commit::String" >> $@
- @echo " commit_short::String" >> $@
- @echo " branch::String" >> $@
- @echo " build_number::Int" >> $@
- @echo " date_string::String" >> $@
- @echo " tagged_commit::Bool" >> $@
- @echo " fork_master_distance::Int" >> $@
- @echo " fork_master_timestamp::Float64" >> $@
- @echo " TAGGED_RELEASE_BANNER::String" >> $@
- @echo "end" >> $@
-
-
- @echo "const BUILD_INFO = BuildInfo( \
- '\"$(version_string)\"', \
- '\"$(commit)\"', \
- '\"$(commit_short)$(dirty)\"', \
- '\"$(git_branch)\"', \
- $(build_number), \
- '\"$(date_string)\"', \
- $(tagged_commit), \
- $(fork_master_distance), \
- $(fork_master_timestamp)., \
- '\"$(TAGGED_RELEASE_BANNER)\"' \
- )" | xargs >> $@
+ @echo "const VERSION_STRING = \"$(JULIA_VERSION)\"" >> $@
+ @echo "const TAGGED_RELEASE_BANNER = \"$(TAGGED_RELEASE_BANNER)\"" >> $@
 
  @# This to ensure that we always rebuild this file, but only when it is modified do we touch build_h.jl,
  @# ensuring we rebuild the system image as infrequently as possible
@@ -106,7 +52,30 @@ endif
  rm -f $@; \
  fi
 
-.PHONY: build_h.jl.phony
+version_git.jl.phony:
+ifneq ($(NO_GIT), 1)
+ @sh version_git.sh > $@
+ @# This to avoid touching git_version.jl when it is not modified,
+ @# so that the system image does not need to be rebuilt.
+ @if ! cmp -s $@ version_git.jl; then \
+ $(call PRINT_PERL,) \
+ mv $@ version_git.jl; \
+ else \
+ rm -f $@; \
+ fi
+else
+ifeq ($(shell [ -f version_git.jl ] && echo "true"), true)
+ @# Give warning if boilerplate git is used
+ @if grep -q "Default output if git is not available" version_git.jl; then \
+ echo "WARNING: Using boilerplate git version info" >&2; \
+ fi
+else
+ $(warning "WARNING: Using boilerplate git version info")
+ @sh version_git.sh NO_GIT > version_git.jl
+endif
+endif
+
+.PHONY: build_h.jl.phony version_git.jl.phony
 
 
 
@@ -115,6 +84,9 @@ clean:
  rm -f pcre_h.jl
  rm -f errno_h.jl
  rm -f build_h.jl
+ rm -f build_h.jl.phony
  rm -f fenv_constants.jl
  rm -f uv_constants.jl
  rm -f file_constants.jl
+ rm -f version_git.jl
+ rm -f version_git.jl.phony

base/methodshow.jl

@@ -80,7 +80,7 @@ function url(m::Method)
  line = m.func.code.line
  line <= 0 || ismatch(r"In\[[0-9]+\]", file) && return ""
  if inbase(M)
- return "https://github.com/JuliaLang/julia/tree/$(Base.BUILD_INFO.commit)/base/$file#L$line"
+ return "https://github.com/JuliaLang/julia/tree/$(Base.GIT_VERSION_INFO.commit)/base/$file#L$line"
  else 
  try
  d = dirname(file)

base/pkg/generate.jl

@@ -60,7 +60,7 @@ function package(
  years: $years
  user: $user
 
- Julia Version $VERSION [$(Base.BUILD_INFO.commit[1:10])]
+ Julia Version $VERSION [$(Base.GIT_VERSION_INFO.short_commit)]
  """
 
  if isnew

base/sysimg.jl

@@ -23,6 +23,7 @@ end
 include("base.jl")
 include("reflection.jl")
 include("build_h.jl")
+include("version_git.jl")
 include("c.jl")
 
 # core operations & types

base/util.jl

@@ -359,8 +359,8 @@ end
 
 function versioninfo(io::IO=STDOUT, verbose::Bool=false)
  println(io, "Julia Version $VERSION")
- if !isempty(BUILD_INFO.commit_short)
- println(io, "Commit $(BUILD_INFO.commit_short) ($(BUILD_INFO.date_string))")
+ if !isempty(GIT_VERSION_INFO.commit_short)
+ println(io, "Commit $(GIT_VERSION_INFO.commit_short) ($(GIT_VERSION_INFO.date_string))")
  end
  if ccall(:jl_is_debugbuild, Bool, ())
  println(io, "DEBUG build")

base/version.jl

@@ -181,28 +181,28 @@ end
 ## julia version info
 
 # Include build number if we've got at least some distance from a tag (e.g. a release)
-build_number = BUILD_INFO.build_number != 0 ? "+$(BUILD_INFO.build_number)" : ""
 try
- global const VERSION = convert(VersionNumber, "$(BUILD_INFO.version_string)$(build_number)")
+ build_number = GIT_VERSION_INFO.build_number != 0 ? "+$(GIT_VERSION_INFO.build_number)" : ""
+ global const VERSION = convert(VersionNumber, "$(VERSION_STRING)$(build_number)")
 catch e
  println("while creating Base.VERSION, ignoring error $e")
  global const VERSION = VersionNumber(0)
 end
 
 function banner(io::IO = STDOUT)
- if BUILD_INFO.tagged_commit
- commit_string = BUILD_INFO.TAGGED_RELEASE_BANNER
- elseif BUILD_INFO.commit == ""
+ if GIT_VERSION_INFO.tagged_commit
+ commit_string = TAGGED_RELEASE_BANNER
+ elseif GIT_VERSION_INFO.commit == ""
  commit_string = ""
  else
- days = int(floor((ccall(:clock_now, Float64, ()) - BUILD_INFO.fork_master_timestamp) / (60 * 60 * 24)))
- if BUILD_INFO.fork_master_distance == 0
- commit_string = "Commit $(BUILD_INFO.commit_short) ($(days) days old master)"
+ days = int(floor((ccall(:clock_now, Float64, ()) - GIT_VERSION_INFO.fork_master_timestamp) / (60 * 60 * 24)))
+ if GIT_VERSION_INFO.fork_master_distance == 0
+ commit_string = "Commit $(GIT_VERSION_INFO.commit_short) ($(days) days old master)"
  else
- commit_string = "$(BUILD_INFO.branch)/$(BUILD_INFO.commit_short) (fork: $(BUILD_INFO.fork_master_distance) commits, $(days) days)"
+ commit_string = "$(GIT_VERSION_INFO.branch)/$(GIT_VERSION_INFO.commit_short) (fork: $(GIT_VERSION_INFO.fork_master_distance) commits, $(days) days)"
  end
  end
- commit_date = BUILD_INFO.date_string != "" ? " ($(BUILD_INFO.date_string))": ""
+ commit_date = GIT_VERSION_INFO.date_string != "" ? " ($(GIT_VERSION_INFO.date_string))": ""
 
  if have_color
  tx = "\033[0m\033[1m" # text