You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

mxe.github.mk 9.2KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187
  1. # This file is part of MXE. See LICENSE.md for licensing information.
  2. # Standardise GitHub downloads and updates
  3. # Download API has three forms:
  4. # Archive:
  5. # url = <owner>/<repo>/archive/<ref>.tar.gz
  6. # dir = <repo>-<ref>
  7. # if <ref> starts with a single `v`, it is removed from dir
  8. #
  9. # Release:
  10. # Manually uploaded distribution tarballs, especially useful for
  11. # autotools packages with generated sources. No universal convention,
  12. # but the default is:
  13. # url = <owner>/<repo>/releases/downloads/<ref>/<repo>-<version>.[archive extension] | tar.gz
  14. # dir = <repo>-<version>
  15. #
  16. #
  17. # Tarball:
  18. # url = <owner>/<repo>/tarball/<ref>/output-file.tar.gz
  19. # dir = <owner>-<repo>-<short sha>
  20. # Filename doesn't matter as we stream the url to a name of our choosing.
  21. #
  22. # The archive API could be used for all packages, however, if the reference
  23. # is any sort of SHA, the full SHA is used for the directory. We could `cd`
  24. # into it without knowing the SHA beforehand, but the directory length would
  25. # be comical in logs etc.
  26. #
  27. # The release API is based on tags but the uploaded tarballs may use
  28. # any naming convention for the filename and subdir, and also other
  29. # archive types e.g. *.xz
  30. #
  31. # The tarball API accepts references to commits or tags, always using the
  32. # short SHA as the directory. In this case, tag tracking packages would have
  33. # to store the SHA (see #1002). However, this only works for lightweight
  34. # tags, not annotated tags that most projects use for releases.
  35. #
  36. # In summary, we have to use all three.
  37. # The tarball API determines the short SHA length used in the directory name.
  38. # Chances of a collision on a "given" commit seem to decrease as the chance
  39. # of "any" collision increases. If that changes in the future, `make update`
  40. # will fix it.
  41. # Currently (2015-12) the API sets the short SHA length to:
  42. GITHUB_SHA_LENGTH := 7
  43. # Packages must set the following metadata:
  44. # Track branch - Tarball API
  45. # GH_CONF := owner/repo/branches/branch
  46. # updates will use the last commit from the specified branch as
  47. # a version string and bypass `sort -V`
  48. #
  49. # Track releases - Release API
  50. # GH_CONF := owner/repo/releases[/latest], tag prefix, tag suffix, tag filter-out, version separator, archive extension
  51. # updates can optionally use the latest non-prerelease with /latest
  52. # or manually specify version numbering based on:
  53. # <tag prefix><s/<version sep>/./version><tag suffix>
  54. #
  55. # Track tags - Archive API
  56. # GH_CONF := owner/repo/tags, tag prefix, tag suffix, tag filter-out, version separator
  57. # updates will construct a version number based on:
  58. # <tag prefix><s/<version sep>/./version><tag suffix>
  59. #
  60. GH_APIS := branches releases tags
  61. # common tag filtering is applied with `grep -v`:
  62. GITHUB_TAG_FILTER := alpha\|beta\|rc
  63. # More complex filters can fall back to `MXE_GET_GH_TAGS` which returns
  64. # a list for post-processing.
  65. # ...and finally, auto-configure packages based on above metadata:
  66. # - `eval` these snippets during PKG_RULE loop (where PKG is in scope).
  67. # - `call` or `eval` from package makefiles requires complex quoting
  68. # and looks out of place.
  69. # - don't redefine manually set standard variables (FILE, SUBDIR, URL, UPDATE)
  70. GH_OWNER = $(word 1,$(subst /,$(space),$(word 1,$(subst $(comma),$(space),$($(PKG)_GH_CONF)))))
  71. GH_REPO = $(word 2,$(subst /,$(space),$(word 1,$(subst $(comma),$(space),$($(PKG)_GH_CONF)))))
  72. GH_API = $(word 3,$(subst /,$(space),$(word 1,$(subst $(comma),$(space),$($(PKG)_GH_CONF)))))
  73. GH_BRANCH = $(and $(filter branches,$(GH_API)),$(word 4,$(subst /,$(space),$(word 1,$(subst $(comma),$(space),$($(PKG)_GH_CONF))))))
  74. GH_LATEST = $(and $(filter releases,$(GH_API)),$(word 4,$(subst /,$(space),$(word 1,$(subst $(comma),$(space),$($(PKG)_GH_CONF))))))
  75. GH_TAG_VARS = $(call rest,$(subst $(comma),$(space)$(__gmsl_aa_magic),$(subst $(space),,$($(PKG)_GH_CONF))))
  76. GH_TAG_PREFIX = $(subst $(__gmsl_aa_magic),,$(word 1,$(GH_TAG_VARS)))
  77. GH_TAG_SUFFIX = $(subst $(__gmsl_aa_magic),,$(word 2,$(GH_TAG_VARS)))
  78. GH_TAG_FILTER = $(subst $(__gmsl_aa_magic),,$(word 3,$(GH_TAG_VARS)))
  79. GH_VERSION_SEP = $(subst $(__gmsl_aa_magic),,$(word 4,$(GH_TAG_VARS)))
  80. GH_ARCHIVE_EXT = $(subst $(__gmsl_aa_magic),,$(word 5,$(GH_TAG_VARS)))
  81. define MXE_SETUP_GITHUB
  82. $(PKG)_GH_OWNER := $(GH_OWNER)
  83. $(PKG)_GH_REPO := $(GH_REPO)
  84. $(PKG)_GH_LATEST := $(if $(GH_LATEST),/latest)
  85. $(PKG)_BRANCH := $(GH_BRANCH)
  86. $(PKG)_TAG_VARS := $(GH_TAG_VARS)
  87. $(PKG)_TAG_PREFIX := $(GH_TAG_PREFIX)
  88. $(PKG)_TAG_SUFFIX := $(GH_TAG_SUFFIX)
  89. $(PKG)_TAG_FILTER := $(GH_TAG_FILTER)
  90. $(PKG)_VERSION_SEP := $(or $(GH_VERSION_SEP),.)
  91. $(PKG)_ARCHIVE_EXT := $(or $(GH_ARCHIVE_EXT),.tar.gz)
  92. $(PKG)_FILE := $(or $($(PKG)_FILE),$(PKG)-$$(filter-out $$(PKG)-,$$($$(PKG)_TAG_PREFIX))$($(PKG)_VERSION)$$($$(PKG)_TAG_SUFFIX)$$($$(PKG)_ARCHIVE_EXT))
  93. $(if $(and $(GH_BRANCH),$(GH_TAG_VARS)),\
  94. $(error $(newline) $(PKG) specifies both branch and tag variables $(newline)))
  95. $(if $(filter-out $(GH_APIS),$(GH_API))$(filter x,x$(GH_API)),\
  96. $(error $(newline) $(PKG) has unknown API in GH_CONF := $($(PKG)_GH_CONF) $(newline)\
  97. must be $(call merge,|,$(GH_APIS))))
  98. $(value MXE_SETUP_GITHUB_$(call uc,$(GH_API)))
  99. endef
  100. define MXE_SETUP_GITHUB_BRANCHES
  101. $(PKG)_SUBDIR := $(or $($(PKG)_SUBDIR),$($(PKG)_GH_OWNER)-$($(PKG)_GH_REPO)-$($(PKG)_VERSION))
  102. $(PKG)_URL := $(or $($(PKG)_URL),https://github.com/$($(PKG)_GH_OWNER)/$($(PKG)_GH_REPO)/tarball/$($(PKG)_VERSION)/$($(PKG)_FILE))
  103. $(PKG)_UPDATE := $(or $($(PKG)_UPDATE),$(call MXE_GET_GH_SHA,$($(PKG)_GH_OWNER)/$($(PKG)_GH_REPO),$($(PKG)_BRANCH)))
  104. endef
  105. define MXE_SETUP_GITHUB_RELEASES
  106. $(PKG)_SUBDIR := $(or $($(PKG)_SUBDIR),$($(PKG)_GH_REPO)-$(if $(call sne,v,$($(PKG)_TAG_PREFIX)),$($(PKG)_TAG_PREFIX))$(subst .,$($(PKG)_VERSION_SEP),$($(PKG)_VERSION))$($(PKG)_TAG_SUFFIX))
  107. $(PKG)_TAG_REF := $(or $($(PKG)_TAG_REF),$($(PKG)_TAG_PREFIX)$(subst .,$($(PKG)_VERSION_SEP),$($(PKG)_VERSION))$($(PKG)_TAG_SUFFIX))
  108. $(PKG)_URL := $(or $($(PKG)_URL),https://github.com/$($(PKG)_GH_OWNER)/$($(PKG)_GH_REPO)/releases/download/$($(PKG)_TAG_REF)/$($(PKG)_SUBDIR)$($(PKG)_ARCHIVE_EXT))
  109. $(PKG)_URL_2 := $(or $($(PKG)_URL_2),https://github.com/$($(PKG)_GH_OWNER)/$($(PKG)_GH_REPO)/archive/$($(PKG)_TAG_REF).tar.gz)
  110. $(PKG)_UPDATE := $(or $($(PKG)_UPDATE),$(call MXE_GET_GH_RELEASE,$($(PKG)_GH_OWNER)/$($(PKG)_GH_REPO)/releases$($(PKG)_GH_LATEST),$($(PKG)_TAG_PREFIX),$($(PKG)_TAG_SUFFIX),$(or $($(PKG)_TAG_FILTER),$(GITHUB_TAG_FILTER)),$($(PKG)_VERSION_SEP)))
  111. endef
  112. define MXE_SETUP_GITHUB_TAGS
  113. $(PKG)_SUBDIR := $(or $($(PKG)_SUBDIR),$($(PKG)_GH_REPO)-$(if $(call sne,v,$($(PKG)_TAG_PREFIX)),$($(PKG)_TAG_PREFIX))$(subst .,$($(PKG)_VERSION_SEP),$($(PKG)_VERSION))$($(PKG)_TAG_SUFFIX))
  114. $(PKG)_TAR_GZ := $(or $($(PKG)_TAR_GZ),$($(PKG)_TAG_PREFIX)$(subst .,$($(PKG)_VERSION_SEP),$($(PKG)_VERSION))$($(PKG)_TAG_SUFFIX))
  115. $(PKG)_URL := $(or $($(PKG)_URL),https://github.com/$($(PKG)_GH_OWNER)/$($(PKG)_GH_REPO)/archive/$($(PKG)_TAR_GZ).tar.gz)
  116. $(PKG)_UPDATE := $(or $($(PKG)_UPDATE),$(call MXE_GET_GH_TAG,$($(PKG)_GH_OWNER)/$($(PKG)_GH_REPO),$($(PKG)_TAG_PREFIX),$($(PKG)_TAG_SUFFIX),$(or $($(PKG)_TAG_FILTER),$(GITHUB_TAG_FILTER)),$($(PKG)_VERSION_SEP)))
  117. endef
  118. # called with owner/repo/releases[/latest],tag prefix, tag suffix, filter-out, version sep
  119. define MXE_GET_GH_RELEASE
  120. $(WGET) -q -O- 'https://github.com/$(strip $(1))' \
  121. | $(SED) -n 's,.*releases/tag/\([^"&]*\)".*,\1,p' \
  122. | $(if $(4),grep -vi '$(strip $(4))') \
  123. | $(SED) -n 's,^$(strip $(2))\([^"]*\)$(strip $(3))$$,\1,p' \
  124. | tr '$(strip $(5))' '.' \
  125. | $(SORT) -V \
  126. | tail -1
  127. endef
  128. # called with owner/repo,branch
  129. define MXE_GET_GH_SHA
  130. $(WGET) -q -O- 'https://api.github.com/repos/$(strip $(1))/git/refs/heads/$(strip $(2))' \
  131. | $(SED) -n 's#.*"sha": "\([^"]\{$(GITHUB_SHA_LENGTH)\}\).*#\1#p' \
  132. | head -1
  133. endef
  134. # called with owner/repo
  135. define MXE_GET_GH_TAGS
  136. $(WGET) -q -O- 'https://github.com/$(strip $(1))/tags' \
  137. | $(SED) -n 's#.*releases/tag/\([^"]*\).*#\1#p'
  138. endef
  139. # called with owner/repo, tag prefix, tag suffix, filter-out, version sep
  140. define MXE_GET_GH_TAG
  141. $(MXE_GET_GH_TAGS) \
  142. | $(if $(4),grep -vi '$(strip $(4))') \
  143. | $(SED) -n 's,^$(strip $(2))\([^"]*\)$(strip $(3))$$,\1,p' \
  144. | tr '$(strip $(5))' '.' \
  145. | $(SORT) -V \
  146. | tail -1
  147. endef
  148. GITHUB_PKGS = $(patsubst %_GH_CONF,%,$(filter %_GH_CONF,$(.VARIABLES)))
  149. # test downloads, updates, and source directory
  150. # make check-gh-conf MXE_PLUGIN_DIRS="`find plugins -name '*.mk' -print0 | xargs -0 -n1 dirname | sort | uniq | tr '\n' ' '`"
  151. # a test of many package updates may hit rate limit of 60/hr
  152. # https://developer.github.com/v3/#rate-limiting
  153. .PHONY: check-gh-conf check-gh-conf-%
  154. check-gh-conf-pkg-%: check-update-package-%
  155. @$(MAKE) -f '$(MAKEFILE)' 'download-only-$(*)' \
  156. REMOVE_DOWNLOAD=true \
  157. MXE_NO_BACKUP_DL=true \
  158. --no-print-directory
  159. @$(PRINTF_FMT) '[prep-src]' '$(*)' | $(RTRIM)
  160. @($(MAKE) -f '$(MAKEFILE)' 'prepare-pkg-source-$(*)') &> '$(LOG_DIR)/$(*)-prep-src'
  161. @rm -rf '$(call TMP_DIR,$(*))'
  162. # secondexpansion here since this file is included before pkg makefiles
  163. .SECONDEXPANSION:
  164. check-gh-conf: $$(addprefix check-gh-conf-pkg-,$$(GITHUB_PKGS))
  165. github-pkgs: $$(GITHUB_PKGS)