[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]
Re: Prepare for encoding changes in Python 3 (issue 573280043 by address
|
From: |
jonas . hahnfeld |
|
Subject: |
Re: Prepare for encoding changes in Python 3 (issue 573280043 by address@hidden) |
|
Date: |
Sun, 24 Nov 2019 03:47:01 -0800 |
Reviewers: lemzwerg, Dan Eble,
Message:
On 2019/11/23 16:45:17, Dan Eble wrote:
> 3. Open midi and musicxml files in binary mode
> Only decode once we are sure that the content is not compressed.
If an XML file is opened as a binary file, will the treatment of
platform-specific line endings become inconvenient for some people?
I hope it does not: Once we know it's really text, the code will call
decode('utf-8') as before.
Do you have a specific case in mind that I should have a look at? I
tested with Python 2.4 and 2.7 on Linux, and it passes a full check +
doc.
Description:
Prepare for encoding changes in Python 3
This is most of the remaining diff for porting to Python 3,
but this also works in Python 2.4 already (even though not
strictly needed).
Individual changes:
1. Encode strings before hashing
Python 3 requires "bytes-like objects". Apparently we can already
do so in Python 2, even though not needed for correctness.
2. Submit bytes-like objects to processes
Encode strings before writing to input and decode result after reading
from output or error stream.
3. Open midi and musicxml files in binary mode
Only decode once we are sure that the content is not compressed.
Please review this at https://codereview.appspot.com/573280043/
Affected files (+28, -25 lines):
M python/book_snippets.py
M scripts/build/output-distance.py
M scripts/lilypond-book.py
M scripts/musicxml2ly.py
Index: python/book_snippets.py
diff --git a/python/book_snippets.py b/python/book_snippets.py
index
3afc411d108d759471be879b675e777669daa748..11cb628e93b4935a139c4bac4f614ad3dfe09659
100644
--- a/python/book_snippets.py
+++ b/python/book_snippets.py
@@ -577,9 +577,9 @@ class LilypondSnippet (Snippet):
# We only want to calculate the hash based on the snippet
# code plus fragment options relevant to processing by
# lilypond, not the snippet + preamble
- hash = md5 (self.relevant_contents (self.ly ()))
+ hash = md5 (self.relevant_contents (self.ly ()).encode
('utf-8'))
for option in self.get_outputrelevant_option_strings ():
- hash.update (option)
+ hash.update (option.encode ('utf-8'))
## let's not create too long names.
self.checksum = hash.hexdigest ()[:10]
@@ -606,7 +606,9 @@ class LilypondSnippet (Snippet):
if self.relevant_contents (existing) != self.relevant_contents
(self.full_ly ()):
warning ("%s: duplicate filename but different contents of
original file,\n\
printing diff against existing file." % filename)
- ly.stderr_write (self.filter_pipe (self.full_ly (), 'diff
-u %s -' % filename))
+ input = self.full_ly ().encode ('utf-8')
+ cmd = 'diff -u %s -' % filename
+ ly.stderr_write (self.filter_pipe (encoded, cmd).decode
('utf-8'))
else:
out = open (filename, 'w')
out.write (self.full_ly ())
@@ -646,12 +648,11 @@ printing diff against existing file." % filename)
try:
if (self.global_options.use_source_file_names
and isinstance (self, LilypondFileSnippet)):
- fout = open (dst, 'w')
- fin = open (src, 'r')
- for line in fin.readlines ():
- fout.write (line.replace (self.basename (),
self.final_basename ()))
- fout.close ()
- fin.close ()
+ content = open (src, 'rb').read ()
+ basename = self.basename ().encode ('utf-8')
+ final_basename = self.final_basename ().encode
('utf-8')
+ content = content.replace (basename, final_basename)
+ open (dst, 'wb').write (content)
else:
try:
os.link (src, dst)
@@ -759,7 +760,7 @@ printing diff against existing file." % filename)
status = 0
output = stdout.read ()
status = stdout.close ()
- err = stderr.read ()
+ err = stderr.read ().decode ('utf-8')
if not status:
status = 0
@@ -769,7 +770,7 @@ printing diff against existing file." % filename)
ly.error (_ ("`%s' failed (%d)") % (cmd, exit_status))
ly.error (_ ("The error log is as follows:"))
ly.stderr_write (err)
- ly.stderr_write (stderr.read ())
+ ly.stderr_write (stderr.read ().decode ('utf-8'))
exit (status)
debug ('\n')
@@ -822,13 +823,13 @@ class LilypondFileSnippet (LilypondSnippet):
LilypondSnippet.__init__ (self, type, match, formatter,
line_number, global_options)
self.filename = self.substring ('filename')
self.contents = open (BookBase.find_file (self.filename,
- global_options.include_path,
global_options.original_dir)).read ()
+ global_options.include_path,
global_options.original_dir), 'rb').read ()
def get_snippet_code (self):
- return self.contents;
+ return self.contents.decode ('utf-8')
def verb_ly (self):
- s = self.contents
+ s = self.get_snippet_code ()
s = re_begin_verbatim.split (s)[-1]
s = re_end_verbatim.split (s)[0]
if not NOGETTEXT in self.option_dict:
@@ -840,7 +841,7 @@ class LilypondFileSnippet (LilypondSnippet):
def ly (self):
name = self.filename
return ('\\sourcefilename \"%s\"\n\\sourcefileline 0\n%s'
- % (name, self.contents))
+ % (name, self.get_snippet_code ()))
def final_basename (self):
if self.global_options.use_source_file_names:
@@ -890,6 +891,7 @@ class MusicXMLFileSnippet (LilypondFileSnippet):
progress (_ ("Converting MusicXML file `%s'...") % self.filename)
ly_code = self.filter_pipe (self.contents, 'musicxml2ly %s --out=-
- ' % opts)
+ ly_code = ly_code.decode ('utf-8')
return ly_code
def ly (self):
@@ -916,20 +918,22 @@ class MusicXMLFileSnippet (LilypondFileSnippet):
if diff_against_existing:
warning (_ ("%s: duplicate filename but different contents
of original file,\n\
printing diff against existing file.") % xmlfilename)
- ly.stderr_write (diff_against_existing)
+ ly.stderr_write (diff_against_existing.decode ('utf-8'))
else:
- out = open (xmlfilename, 'w')
+ out = open (xmlfilename, 'wb')
out.write (self.contents)
out.close ()
# also write the converted lilypond
filename = path + '.ly'
if os.path.exists (filename):
- diff_against_existing = self.filter_pipe (self.full_ly
(), 'diff -u %s -' % filename)
+ input = self.full_ly ().encode ('utf-8')
+ cmd = 'diff -u %s -' % filename
+ diff_against_existing = self.filter_pipe (encoded, cmd).decode
('utf-8')
if diff_against_existing:
warning (_ ("%s: duplicate filename but different contents
of converted lilypond file,\n\
printing diff against existing file.") % filename)
- ly.stderr_write (diff_against_existing)
+ ly.stderr_write (diff_against_existing.decode ('utf-8'))
else:
out = open (filename, 'w')
out.write (self.full_ly ())
Index: scripts/build/output-distance.py
diff --git a/scripts/build/output-distance.py
b/scripts/build/output-distance.py
index
69ea0a9819cff8e6455daace8d6bcfc8a0605a42..2148dbe688a308b6b662dbb8087b6f9cf6c978c1
100755
--- a/scripts/build/output-distance.py
+++ b/scripts/build/output-distance.py
@@ -587,7 +587,7 @@ class MidiFileLink (TextFileCompareLink):
def get_content (self, oldnew):
import midi
- data = FileCompareLink.get_content (self, oldnew)
+ data = open (oldnew, 'rb').read ()
midi = midi.parse (data)
tracks = midi[1]
Index: scripts/lilypond-book.py
diff --git a/scripts/lilypond-book.py b/scripts/lilypond-book.py
index
73e5ad2079b2561a27f4d8ac8fba3b214f63746b..971f12e99c533459dcb081079871b48860ba4cd5
100644
--- a/scripts/lilypond-book.py
+++ b/scripts/lilypond-book.py
@@ -414,7 +414,7 @@ def process_snippets (cmd, snippets,
name = os.path.join (lily_output_dir,
'snippet-names-%d.ly' % checksum)
logfile = name.replace('.ly', '')
- open (name, 'wb').write (contents)
+ open (name, 'wb').write (contents.encode ('utf-8'))
system_in_directory (' '.join ([cmd, ly.mkarg (name.replace
(os.path.sep, '/'))]),
lily_output_dir,
Index: scripts/musicxml2ly.py
diff --git a/scripts/musicxml2ly.py b/scripts/musicxml2ly.py
index
29a3fcb1b810965139b65d97cc7b98621017d547..1a23236a8982ff83097b466a57c5f8a994c5279a
100755
--- a/scripts/musicxml2ly.py
+++ b/scripts/musicxml2ly.py
@@ -3102,14 +3102,13 @@ def read_musicxml(filename, compressed, use_lxml):
sys.stdin = os.fdopen(sys.stdin.fileno(), 'rb', 0) # Make
sys.stdin binary
bytes_read = sys.stdin.read(8192)
while bytes_read:
- for b in bytes_read:
- tmp.write(b)
+ tmp.write(bytes_read)
bytes_read = sys.stdin.read(8192)
z = zipfile.ZipFile(tmp, "r")
else:
ly.progress(_("Input file %s is compressed, extracting raw
MusicXML data") % filename, True)
z = zipfile.ZipFile(filename, "r")
- container_xml = z.read("META-INF/container.xml")
+ container_xml = z.read("META-INF/container.xml").decode("utf-8")
if not container_xml:
return None
container = read_xml(StringIO.StringIO(container_xml), use_lxml)
@@ -3123,7 +3122,7 @@ def read_musicxml(filename, compressed, use_lxml):
if len(rootfile_list) > 0:
mxml_file = getattr(rootfile_list[0], 'full-path', None)
if mxml_file:
- raw_string = z.read(mxml_file)
+ raw_string = z.read(mxml_file).decode('utf-8')
if raw_string:
io_object = StringIO.StringIO(raw_string)