[Top][All Lists]
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]
Re-write of documentation tree build process
|
From: |
John Mandereau |
|
Subject: |
Re-write of documentation tree build process |
|
Date: |
Thu, 14 Dec 2006 23:09:11 +0100 |
Hello all,
Here's a (long-awaited) patch against the stable/2.10 branch.
'make web' can now generate two documentation trees in
<src-dir>/out-www:
- one in online-root/, where .png and .html extensions are stripped in
HTML files, for hosting on a web site with content negociation.
- the other in /offline-root, for browsing the docs on a hard disk.
Only one of these trees or both can be generated, depending on the
FINAL_TARGETS variable:
make FINAL_TARGETS=offline web # same as 'make web'
make FINAL_TARGETS=online web
only builds one tree, and
make FINAL_TARGETS="offline online" web
builds both.
If this can be applied and a possible link to the French tutorial in
index.html.in is added, Jan and Han-Wen may want to release 2.10.3 or
2.10.2-2, as the current docball is broken because of .html stripping
(maybe the 2.11 docball is broken too).
Cheers,
--
John Mandereau <address@hidden>
diff --git a/GNUmakefile.in b/GNUmakefile.in
index 7f3dee5..37c47e4 100644
--- a/GNUmakefile.in
+++ b/GNUmakefile.in
@@ -49,7 +49,7 @@ doc:
install-WWW:
-$(INSTALL) -m 755 -d $(DESTDIR)$(webdir)
- cp -a $(outdir)/web-root/ $(DESTDIR)$(webdir)/
+ cp -a $(outdir)/offline-root/ $(DESTDIR)$(webdir)/
$(MAKE) -C Documentation/user local-install-WWW
$(MAKE) -C Documentation/user install-info
@@ -73,11 +73,12 @@ local-install:
final-install:
@true
-web-ext = html midi pdf png txt ly signature
+common-web-ext = midi|pdf|png|txt|ly|signature
-# For docball, issue `make web CONTENT_NEGOTIATION='
-CONTENT_NEGOTIATION = --content-negotiation
-footify = $(PYTHON) $(step-bindir)/add-html-footer.py --name $(PACKAGE_NAME)
--version $(TOPLEVEL_VERSION) $(CONTENT_NEGOTIATION)
+# For online docs with content negotiation, issue `make web
FINAL_TARGETS=online'
+# For both online and offline docs, issue `make web FINAL_TARGETS="offline
online"'
+FINAL_TARGETS = offline
+footify = $(PYTHON) $(step-bindir)/add-html-footer.py --name $(PACKAGE_NAME)
--version $(TOPLEVEL_VERSION) --targets "$(foreach i, $(FINAL_TARGETS),
$(i)-root:$(i) )"
footifymail =
MAILADDRESS='http://post.gmane.org/post.php?group=gmane.comp.gnu.lilypond.bugs'
@@ -86,33 +87,14 @@ local-WWW-post:
# need UTF8 setting in case this is hosted on a website.
echo -e 'AddDefaultCharset utf-8\nAddCharset utf-8 .html\nAddCharset
utf-8 .en\nAddCharset utf-8 .nl\nAddCharset utf-8 .txt\n' >
$(top-build-dir)/.htaccess
$(PYTHON) $(buildscript-dir)/mutopia-index.py -o
$(outdir)/examples.html input/
+ $(foreach i, $(FINAL_TARGETS), rm -rf $(outdir)/$(i)-root &&) true
+ $(PYTHON) $(step-bindir)/build-trees.py --input-root
Documentation,input --lose-dir out-www --process-dir out-www --common-files
'.*[.](?:$(common-web-ext)$$)' --specific-files '.*[.]html$$' --exclude-dir
'(fr|po|out)(/|$$)' --exclude-files 'lily-[0-9].*.pdf' --dump-sfl
$(outdir)/html-list --target-pattern $(outdir)/%s-root $(FINAL_TARGETS)
echo '<META HTTP-EQUIV="refresh"
content="0;URL=Documentation/index.html">' > $(outdir)/index.html
echo '<html><body>Redirecting to the documentation
index...</body></html>' >> $(outdir)/index.html
-
- cd $(top-build-dir) && $(FIND) . -name '*.html' -print | $(footifymail)
xargs $(footify)
-
- cd $(top-build-dir) && find Documentation input \
- $(web-ext:%=-path '*/out-www/*.%' -or) -type l \
- | grep -v 'lily-[0-9a-f]*.*pdf' \
- | grep -v '/fr/' \
- > $(outdir)/weblist
- ls $(outdir)/*.html >> $(outdir)/weblist
-
-## urg: this is too hairy, should write a python script to do this.
-
-## rewrite file names so we lose out-www
- rm -rf $(outdir)/web-root/
- mkdir $(outdir)/web-root/
-## urg slow.
- cat $(outdir)/weblist | (cd $(top-build-dir); tar -cf- -T- ) | \
- tar -C $(outdir)/web-root/ -xf -
- for dir in $(outdir)/web-root/ ; do \
- cd $$dir && \
- for a in `find . -name out-www`; do \
- rsync -a --link-dest $$a/ $$a/ $$a/.. ; \
- rm -rf $$a ; \
- done \
- done
+ cd $(outdir) && ls --format=single-column *.html >> html-list
+ $(foreach i, $(FINAL_TARGETS), echo $(TOPLEVEL_VERSION) >
$(outdir)/$(i)-root/VERSION && \
+ cp $(outdir)/*.html $(outdir)/$(i)-root/ &&) true
+ cd $(outdir) && cat html-list | $(footifymail) xargs $(footify)
tree-prefix = $(outdir)
tree-bin = $(tree-prefix)/bin
diff --git a/stepmake/bin/add-html-footer.py b/stepmake/bin/add-html-footer.py
index 08ff5ff..9a28237 100644
--- a/stepmake/bin/add-html-footer.py
+++ b/stepmake/bin/add-html-footer.py
@@ -13,7 +13,7 @@ import getopt
index_url=''
top_url=''
changelog_file=''
-content_negotiation = False
+targets = []
package_name = ''
package_version = ''
@@ -69,7 +69,10 @@ Add header, footer and top of ChangLog f
Options:
--changelog=FILE use FILE as ChangeLog [ChangeLog]
- --content-negotiation strip .html and .png from urls
+ --targets=TARGETS set targets
+ TARGETS is a space-separated list of PATH:TYPE
+ where PATH is the path prefix of HTML-FILE
+ TYPE is one of online, offline
--footer=FILE use FILE as footer
--header=FILE use FILE as header
-h, --help print this help
@@ -82,15 +85,15 @@ Options:
(options, files) = getopt.getopt(sys.argv[1:], 'h', [
'changelog=', 'footer=', 'header=', 'help', 'index=',
- 'name=', 'content-negotiation', 'version='])
+ 'name=', 'targets=', 'version='])
for opt in options:
o = opt[0]
a = opt[1]
if o == '--changelog':
changelog_file = a
- elif o == '--content-negotiation':
- content_negotiation = True
+ elif o == '--targets':
+ targets = filter (lambda l: l != [''], map (lambda s: re.split (':',
s), re.split (' ', a)))
elif o == '--footer':
footer_file = a
elif o == '--header':
@@ -107,6 +110,47 @@ for opt in options:
else:
raise 'unknown opt ', o
+if targets == []:
+ targets = ['', 'offline']
+
+def lang_file_name (p, langext, ext):
+ if langext != '':
+ return p + '.' + langext + ext
+ return p + ext
+
+class language_def:
+ def __init__ (self, code, name, webext=None):
+ self.code = code
+ self.name = name
+ if webext == None:
+ self.webext = self.code
+ else:
+ self.webext = webext
+ def file_name (self, prefix, ext):
+ return lang_file_name (prefix, self.webext, ext)
+
+C = 'site'
+LANGUAGES = {
+ C: language_def ('en', 'English', ''),
+# 'nl': lang ('nl', 'Nederlands'),
+ 'fr': language_def ('fr', 'French')
+ }
+
+# build dictionnary of available translations of each page
+html_re = re.compile ('(.*?)(?:[.]([^/.]*))?.html')
+pages_dict = {}
+for f in files:
+ m = html_re.match (f)
+ if m:
+ g = m.groups()
+ if len (g) <= 1 or g[1] == None:
+ e = ''
+ else:
+ e = g[1]
+ if not g[0] in pages_dict.keys():
+ pages_dict[g[0]] = [e]
+ else:
+ pages_dict[g[0]].append (e)
def compose (default, file):
s = default
@@ -171,11 +215,14 @@ def remove_self_ref (s):
m = pat.search (s)
return s
-def do_file (f):
- s = open (f).read()
+def do_file (prefix, lang_ext):
+ f = lang_file_name (prefix, lang_ext, '.html')
+ in_f = open (os.path.join (targets[0][0], f))
+ s = in_f.read()
+ in_f.close()
+
s = re.sub ('%', '%%', s)
-
-
+
if re.search (header_tag, s) == None:
body = '<BODY BGCOLOR=WHITE TEXT=BLACK>'
s = re.sub ('(?i)<body>', body, s)
@@ -200,8 +247,13 @@ def do_file (f):
else:
s = s + footer_tag + footer + '\n'
- s = i18n (f, s)
-
+ page_flavors = i18n (prefix, lang_ext, s, targets)
+ else: # urg, all flavors have to be made
+ page_flavors = {}
+ for t in targets:
+ for e in map (lambda l: LANGUAGES[l].webext, LANGUAGES.keys()):
+ if not e in pages_dict[prefix]:
+ page_flavors[os.path.join (t[0], lang_file_name (prefix,
e, '.html'))] = s
#URUGRGOUSNGUOUNRIU
index = index_url
top = top_url
@@ -217,7 +269,7 @@ def do_file (f):
versiontup = string.split(package_version, '.')
branch_str = 'stable-branch'
- if string.atoi ( versiontup[1]) % 2:
+ if int ( versiontup[1]) % 2:
branch_str = 'development-branch'
wiki_page = ('v%s.%s-' % (versiontup[0], versiontup[1]) + f)
@@ -236,27 +288,28 @@ def do_file (f):
subst = globals ()
subst.update (locals())
- s = s % subst
+ for k in page_flavors.keys():
+ page_flavors[k] = page_flavors[k] % subst
# urg
# maybe find first node?
fallback_web_title = '-- --'
-
# ugh, python2.[12] re is broken.
#m = re.match ('.*?<title>\(.*?\)</title>', s, re.DOTALL)
m = re.match ('[.\n]*?<title>([.\n]*?)</title>', s)
if m:
fallback_web_title = m.group (1)
- s = re.sub ('@WEB-TITLE@', fallback_web_title, s)
-
- s = remove_self_ref (s)
-
- # remove info's annoying's indication of referencing external document
- s = re.sub (' \((lilypond|lilypond-internals|music-glossary)\)</a>',
- '</a>', s)
-
- open (f, 'w').write (s)
+ for k in page_flavors.keys():
+ page_flavors[k] = re.sub ('@WEB-TITLE@', fallback_web_title,
page_flavors[k])
+
+ page_flavors[k] = remove_self_ref (page_flavors[k])
+ # remove info's annoying's indication of referencing external document
+ page_flavors[k] = re.sub ('
\((lilypond|lilypond-internals|music-glossary)\)</a>',
+ '</a>', page_flavors[k])
+ out_f = open (k, 'w')
+ out_f.write (page_flavors[k])
+ out_f.close()
localedir = 'out/locale'
@@ -270,13 +323,6 @@ except:
return s
underscore = _
-C = 'site'
-LANGUAGES = (
- (C, 'English'),
- ('nl', 'Nederlands'),
- ('fr', 'French')
- )
-
language_available = _ ("Other languages: %s.") % "%(language_menu)s"
browser_language = _ ("Using <A HREF='%s'>automatic language selection</A>.") \
% "%(root_url)sabout/browser-language"
@@ -289,59 +335,69 @@ LANGUAGES_TEMPLATE = '''\
</P>
''' % vars ()
-def file_lang (file, lang):
- (base, ext) = os.path.splitext (file)
- base = os.path.splitext (base)[0]
- if lang and lang != C:
- return base + '.' + lang + ext
- return base + ext
-
-def i18n (file_name, page):
+def i18n (prefix, lang_ext, page, targets):
# ugh
root_url = "/web/"
+ file_name = lang_file_name (prefix, lang_ext, '.html')
base_name = os.path.basename (file_name)
- lang = C
- m = re.match ('.*[.]([^/.]*).html', file_name)
- if m:
- lang = m.group (1)
-
# Find available translations of this page.
- available = filter (lambda x: lang != x[0] \
- and os.path.exists (file_lang (file_name, x[0])),
- LANGUAGES)
-
- # Strip .html, .png suffix for auto language selection (content
- # negotiation). The menu must keep the full extension, so do
- # this before adding the menu.
- if content_negotiation:
- page = re.sub
('''(href|src)=[\'"]([^/][.]*[^.:\'"]*)(.html|.png)(#[^"\']*|)[\'"]''',
- '\\1="\\2\\4"', page)
-
+ available = []
+ missing = []
+ for l in LANGUAGES.keys():
+ e = LANGUAGES[l].webext
+ if lang_ext != e:
+ if e in pages_dict[prefix]:
+ available.append (LANGUAGES[l])
+ elif lang_ext == '': # write English version of missing translated
pages
+ missing.append (e)
+
+ page_flavors = {}
+ for t in targets:
+ if t[1] == 'online':
+ # Strip .html, .png suffix for auto language selection (content
+ # negotiation). The menu must keep the full extension, so do
+ # this before adding the menu.
+ page_flavors[os.path.join (t[0], file_name)] = re.sub (
+
'''(href|src)=[\'"]([^/][.]*[^.:\'"]*)(.html|.png)(#[^"\']*|)[\'"]''',
+ '\\1="\\2\\4"', page)
+ elif t[1] == 'offline':
+ if lang_ext == '':
+ page_flavors[os.path.join (t[0], file_name)] = page
+ for e in missing:
+ page_flavors[os.path.join (t[0], lang_file_name (prefix,
e, '.html'))] = re.sub (
+
'''href=[\'"]([^/][.]*[^.:\'"]*)(.html)(#[^"\']*|)[\'"]''',
+ 'href="\\1.' + e + '\\2\\3"', page)
+ else:
+ page_flavors[os.path.join (t[0], file_name)] = re.sub (
+ '''href=[\'"]([^/][.]*[^.:\'"]*)(.html)(#[^"\']*|)[\'"]''',
+ 'href="\\1.' + lang_ext + '\\2\\3"', page)
+
# Add menu after stripping: must not have autoselection for language menu.
language_menu = ''
- for (prefix, name) in available:
- lang_file = file_lang (base_name, prefix)
+ for lang in available:
+ lang_file = lang.file_name (os.path.basename (prefix), '.html')
if language_menu != '':
language_menu += ', '
- language_menu += '<a href="%(lang_file)s">%(name)s</a>' % vars ()
+ language_menu += '<a href="%s">%s</a>' % (lang_file, lang.name)
languages = ''
if language_menu:
languages = LANGUAGES_TEMPLATE % vars ()
# Put language menu before '</body>' and '</html>' tags
- if re.search ('(?i)</body', page):
- page = re.sub ('(?i)</body>', languages + '</BODY>', page, 1)
- elif re.search ('(?i)</html', page):
- page = re.sub ('(?i)</html>', languages + '</HTML>', page, 1)
- else:
- page = page + languages
-
- return page
+ for k in page_flavors.keys():
+ if re.search ('(?i)</body', page_flavors[k]):
+ page_flavors[k] = re.sub ('(?i)</body>', languages + '</BODY>',
page_flavors[k], 1)
+ elif re.search ('(?i)</html', page_flavors[k]):
+ page_flavors[k] = re.sub ('(?i)</html>', languages + '</HTML>',
page_flavors[k], 1)
+ else:
+ page_flavors[k] += languages
-for f in files:
- do_file (f)
+ return page_flavors
+for page in pages_dict.keys():
+ for e in pages_dict[page]:
+ do_file (page, e)
diff --git a/stepmake/bin/build-trees.py b/stepmake/bin/build-trees.py
new file mode 100644
index 0000000..e0ef64e
--- /dev/null
+++ b/stepmake/bin/build-trees.py
@@ -0,0 +1,144 @@
address@hidden@
+
+"""
+Build trees for different targets
+"""
+import re
+import sys
+import os
+import getopt
+
+
+
+def help ():
+ sys.stdout.write (r"""Usage: build-trees [OPTIONS]... TARGETS
+Build trees for different targets by hardlinking input trees.
+
+Options:
+ --input-root=DIR use DIR as input tree root (default is current
directory,
+ multiple roots may be specified with a comma
separator)
+ --target-pattern=STRING use STRING as target root directory name pattern
+ --lose-dir=PATTERN lose directories whose name matches PATTERN in
copies
+ (write its content to parent)
+ --process-dir=PATTERN only process files in directories whose name
matches PATTERN
+ --common-files=PATTERN filters files commmon to all trees
+ (they are hardlinked instead of being copied).
+ --specific-files=PATTERN filters files specific to different trees
+ (regular files are only hardlinked to first
target).
+ --exclude-dir=PATTERN don't recurse into directories whose name matches
PATTERN
+ --exclude-files=PATTERN exclude files whose name matches PATTERN
+ --dump-sfl=FILE dump specific files list to FILE
+ -h, --help print this help
+
+PATTERN should be a Python regular expression.
+Common and specific files which are symlinks are always copied to all targets.
+""")
+ sys.exit (0)
+
+(options, targets) = getopt.getopt(sys.argv[1:], 'h', [
+ 'input-root=', 'lose-dir=', 'common-files=', 'help', 'specific-files=',
+ 'exclude-dir=', 'exclude-files=', 'dump-sfl=', 'process-dir=',
'target-pattern='])
+
+input_roots = []
+target_pattern = ''
+
+common_f = '.*'
+specific_f = '.*'
+excluded_d = ''
+excluded_f = ''
+process_d = '.*'
+lost_d_names = ''
+sfl_dump = ''
+
+#file_list_re = re.compile (r'(?:^| )(?:[^ ]*|".*?")(?: |$)')
+
+for opt in options:
+ o = opt[0]
+ a = opt[1]
+ if o == '--input-root':
+ input_roots = re.split (',', a)
+ elif o == '--lose-dir':
+ lost_d_names = a
+ elif o == '--common-files':
+ common_f = a
+ elif o == '--specific-files':
+ specific_f = a
+ elif o == '-h' or o == '--help':
+ help ()
+ elif o == '--exclude-dir':
+ excluded_d = a
+ elif o == '--exclude-files':
+ excluded_f = a
+ elif o == '--process-dir':
+ process_d = a
+ elif o == '--target-pattern':
+ target_pattern = a
+ elif o == '--dump-sfl':
+ sfl_dump = a
+ else:
+ raise 'unknown opt ', o
+
+if input_roots == []:
+ input_roots = ['.']
+
+for s in ['common_f', 'specific_f', 'excluded_d',
+ 'excluded_f', 'process_d', 'lost_d_names']:
+ exec s + '_re = re.compile (' + s + ')'
+strip_lost_d_names_re = re.compile ('/(?:' + lost_d_names + ')')
+slash_re = re.compile ('/')
+
+
+if '%s' in target_pattern:
+ target_dirs = map (lambda s: target_pattern % s, targets)
+else:
+ target_dirs = map (lambda s: target_pattern + s, targets)
+
+for dir in target_dirs:
+ os.mkdir (dir)
+
+path_strip = lambda bigpath, root: bigpath[:len (root)]
+
+def new_link_path (link, dir, r):
+ l = slash_re.split (link)
+ d = slash_re.split (dir)
+ i = 0
+ while l[i] == '..':
+ if r.match (d[i]):
+ del l[i]
+ else:
+ i += 1
+ return reduce (lambda s1, s2: s1 + '/' + s2,
+ filter (lambda x: not r.match (x), l))
+
+if sfl_dump:
+ sfl = open (sfl_dump, 'w')
+
+for d in input_roots:
+ for in_dir, dirs, files in os.walk(d):
+ i = 0
+ while i < len(dirs):
+ if excluded_d_re.search (dirs[i]):
+ del dirs[i]
+ else:
+ i += 1
+ out_dir = strip_lost_d_names_re.sub ('', in_dir)
+ if not lost_d_names_re.match (os.path.basename (in_dir)):
+ for t in target_dirs:
+ os.mkdir (os.path.join (t, out_dir))
+ if not process_d_re.search (in_dir):
+ continue
+ for f in filter (lambda s: not excluded_f_re.match (s), files):
+ in_f = os.path.join (in_dir, f)
+ if os.path.islink (in_f): # all symlinks are assumed to be
relative and to point to files in the input trees
+ link_path = new_link_path (os.readlink (in_f), in_dir,
lost_d_names_re)
+ for t in target_dirs:
+ os.symlink (link_path, os.path.join (t, out_dir, f))
+ elif specific_f_re.match (f):
+ os.link (in_f, os.path.join (target_dirs[0], out_dir, f)) #
only hardlink specific file in first target
+ if sfl_dump:
+ sfl.write (os.path.join (out_dir, f) + '\n')
+ elif common_f_re.match (f):
+ for t in target_dirs:
+ os.link (in_f, os.path.join (t, out_dir, f))
+if sfl_dump:
+ sfl.close()
- Re-write of documentation tree build process,
John Mandereau <=
- Re: Re-write of documentation tree build process, Han-Wen Nienhuys, 2006/12/15
- Re: Re-write of documentation tree build process, John Mandereau, 2006/12/17
- Re: Re-write of documentation tree build process, John Mandereau, 2006/12/20
- Re: Re-write of documentation tree build process, Han-Wen Nienhuys, 2006/12/20
- Re: Re-write of documentation tree build process, John Mandereau, 2006/12/21
- Re: Re-write of documentation tree build process, Juergen Reuter, 2006/12/21
- Re: Re-write of documentation tree build process, Jan Nieuwenhuizen, 2006/12/21
- Re: Re-write of documentation tree build process, John Mandereau, 2006/12/23