Mirror
/
yoga
mirror of https://github.com/wanadev/yoga.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Normalization for paths and textures

This commit is contained in:
Alexis Breust 2018-05-07 11:20:11 +02:00
parent a37b27255b
commit 2fb30576ba
5 changed files with 163 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
requirements.txt
View File

@ -32,5 +32,6 @@ Sphinx==1.7.1
sphinx-rtd-theme==0.2.4
sphinxcontrib-websupport==1.0.1
typing==3.6.4
unidecode==1.0.22
urllib3==1.22
zopflipy==1.0

4
test/test_model.py
View File

@ -26,7 +26,7 @@ class Test_optimize(object):
assert output.read().startswith(_MAGIC_GLB)
def test_input_bytesio(self):
with pytest.raises(RuntimeError):
with pytest.raises(ValueError):
input_ = io.BytesIO(open("test/models/model.fbx", "rb").read())
output = io.BytesIO()
yoga.model.optimize(input_, output)
@ -42,7 +42,7 @@ class Test_optimize(object):
assert output.read().startswith(_MAGIC_GLB)
def test_textures_empty_dictionary(self):
with pytest.raises(RuntimeError):
with pytest.raises(ValueError):
input_ = open("test/models/model.fbx", "rb")
output = io.BytesIO()
yoga.model.optimize(input_, output, {}, {})

73
test/test_model_helpers.py Normal file
View File

@ -0,0 +1,73 @@
# coding=utf8
import pytest
from yoga.model import helpers
class Test_model_helpers(object):
def test_normalize_path(self):
assert helpers.normalize_path(
u"images/texture.png") == "images/texture.png"
assert helpers.normalize_path(
u"./images/texture.png") == "images/texture.png"
assert helpers.normalize_path(
u".\\images\\texture.png") == "images/texture.png"
assert helpers.normalize_path(
u"./images\\texture.png") == "images/texture.png"
assert helpers.normalize_path(
u".\\images/texture.png") == "images/texture.png"
assert helpers.normalize_path(
u"../images/texture.png") == "images/texture.png"
assert helpers.normalize_path(
u"..\\images\\texture.png") == "images/texture.png"
assert helpers.normalize_path(
u"./images/subfolder/../texture.png") == "images/texture.png"
assert helpers.normalize_path(
u"./images/sub1\\sub2/../../texture.png") == "images/texture.png"
assert helpers.normalize_path(
u"./images/texture.png") == "images/texture.png"
assert helpers.normalize_path(
u"/images/texture.png") == "images/texture.png"
assert helpers.normalize_path(
u"C:\\images\\texture.png") == "images/texture.png"
assert helpers.normalize_path(
u"somE_valid-caractères of files.png") == "some_valid-caracteres of files.png" # noqa
def test_normalize_textures(self):
textures = helpers.normalize_textures({
"./images\\texture-1.png": True,
"images/texture-2.png": True,
})
assert "images/texture-1.png" in textures
assert "images/texture-2.png" in textures
with pytest.raises(ValueError):
helpers.normalize_textures({
"images/texture.png": True,
"./images/texture.png": True,
})
def test_find_valid_texture_path(self):
textures = dict({
"images/texture.png": True,
"images/texture.jpg": True,
"other_images/texture.jpg": True,
"texture.gif": True,
})
assert helpers.find_valid_texture_path(
"images/texture.png", textures) == "images/texture.png"
assert helpers.find_valid_texture_path(
"images/texture.jpg", textures) == "images/texture.jpg"
assert helpers.find_valid_texture_path(
"texture.png", textures) == "images/texture.png"
with pytest.raises(ValueError):
helpers.find_valid_texture_path("texture.jpg", textures)
with pytest.raises(ValueError):
helpers.find_valid_texture_path("non-existing.png", textures)
with pytest.raises(ValueError):
helpers.find_valid_texture_path("exture.png", textures)

6
yoga/model/__init__.py
View File

@ -6,6 +6,12 @@ import os.path
def optimize(input_file, output_file, options={}, textures=None):
# TODO: Make a effective documentation.
# The textures arguments should be a dictionary that maps
# paths to bytes. When not None, there will be no file system
# reads in order to find referenced textures. We will
# look into that dictionary instead.
model_options = normalize_options(options)
image_options = extract_image_options(options)

115
yoga/model/helpers.py
View File

@ -1,11 +1,61 @@
from ._assimp import ffi
import io
import re
import os.path
import unidecode
import yoga.image
def normalize_path(path):
# Expects a unicode path, returns a ascii one.
# Paths are normalized to a standard linux relative path,
# without a point, and lowercase.
# That is to say /images\subfolder/..\texture.png -> images/texture.png
# It does not correspond to an effective path,
# as the backslashes on linux are wrongly seen as separators.
# This function is meant to give a standard output.
path = unidecode.unidecode(path)
split_path = re.findall(r"[\w\s\-_.:]+", path)
normalized_path = ""
ignored_folders = 0
for i, name in enumerate(reversed(split_path)):
if name == "." or name[-1:] == ":":
continue
elif name == "..":
ignored_folders += 1
elif ignored_folders > 0:
ignored_folders -= 1
elif i == 0:
normalized_path = name
else:
normalized_path = name + "/" + normalized_path
normalized_path = normalized_path.lower()
return normalized_path
def normalize_textures(textures):
if textures is None:
return None
# Normalizes all the paths in the texture dict.
normalized_textures = dict()
for path in textures:
normalized_path = normalize_path(path.decode("utf-8"))
if normalized_path in normalized_textures:
raise ValueError("Multiple textures are resolved to the same path %s." % normalized_path) # noqa
normalized_textures[normalized_path] = textures[path]
return normalized_textures
def find_valid_path(path, root_path):
# Note: we cannot use normalized paths here,
# because we need to find a file on the system.
tested_path = path
if os.path.isfile(tested_path):
return tested_path
@ -37,34 +87,28 @@ def find_valid_path(path, root_path):
if os.path.isfile(tested_path):
return tested_path
raise RuntimeError(
raise ValueError(
"Cannot resolve file %s, root_path is %s"
% (path, root_path)
)
def find_valid_texture_path(path, textures):
# In textures, all paths are supposed to be relative
# The path and the textures' paths are supposed to have
# already been normalized.
tested_path = path
if tested_path in textures:
return tested_path
split_path = reversed(path.split("/"))
split_paths = map(lambda p: p.split("/"), textures.keys())
tested_path = os.path.basename(path)
if tested_path in textures:
return tested_path
for i, name in enumerate(split_path):
split_paths = filter(lambda sp: len(sp) > i and sp[-(i+1)] == name, split_paths) # noqa
path = path.replace("\\", "/")
if len(split_paths) == 0:
break
elif len(split_paths) == 1:
return "/".join(split_paths[0])
tested_path = path
if tested_path in textures:
return tested_path
tested_path = os.path.basename(path)
if tested_path in textures:
return tested_path
raise RuntimeError(
raise ValueError(
"Cannot resolve file %s within the textures dictionary"
% (path)
)
@ -72,7 +116,8 @@ def find_valid_texture_path(path, textures):
def model_embed_images(images, images_bytes,
optimize_textures, root_path, image_options, textures):
optimized_images = {}
optimized_textures = {}
normalized_textures = normalize_textures(textures)
image = images
while image:
@ -80,33 +125,35 @@ def model_embed_images(images, images_bytes,
continue
image_path = ffi.string(image.path).decode("utf-8")
valid_image_path = None
image_io = None
# If textures exists, we don't look for files on the file system
if textures is not None:
image_path = find_valid_texture_path(image_path, textures)
if normalized_textures is not None:
valid_image_path = normalize_path(image_path)
valid_image_path = find_valid_texture_path(valid_image_path, normalized_textures) # noqa
else:
image_path = find_valid_path(image_path, root_path)
image_path = os.path.abspath(image_path)
valid_image_path = find_valid_path(image_path, root_path)
valid_image_path = os.path.abspath(valid_image_path)
# If image_path have already been seen, do not reoptimize...
if image_path in optimized_images:
optimized_image = optimized_images[image_path]
image.bytes_length = optimized_image.bytes_length
image.bytes = optimized_image.bytes
image.id = optimized_image.id
if valid_image_path in optimized_textures:
optimized_texture = optimized_textures[valid_image_path]
image.bytes_length = optimized_texture.bytes_length
image.bytes = optimized_texture.bytes
image.id = optimized_texture.id
image = image.next
continue
# Get the bytes indeed
if textures is not None:
image_io = textures[image_path]
image_io = textures[valid_image_path]
else:
image_io = io.BytesIO(open(image_path, "rb").read())
image_io = io.BytesIO(open(valid_image_path, "rb").read())
# Optimizing the texture if requested
if optimize_textures:
print("Optimizing texture %s..." % image_path)
print("Optimizing texture %s..." % valid_image_path)
output_io = io.BytesIO()
yoga.image.optimize(image_io, output_io, image_options)
image_io = output_io
@ -118,11 +165,11 @@ def model_embed_images(images, images_bytes,
image_bytes_c = ffi.new("char[%d]" % len(image_bytes), image_bytes)
image.bytes_length = len(image_bytes)
image.bytes = image_bytes_c
image.id = len(optimized_images)
image.id = len(optimized_textures)
optimized_images[image_path] = image
optimized_textures[valid_image_path] = image
image = image.next
# @note Save the bytes to a dictionnary so that the garbage collector
# does not occur before exporting the scene a bit later
images_bytes[image_path] = image_bytes_c
images_bytes[valid_image_path] = image_bytes_c