001/*
002 * (C) Copyright 2006-2016 Nuxeo SA (http://nuxeo.com/) and others.
003 *
004 * Licensed under the Apache License, Version 2.0 (the "License");
005 * you may not use this file except in compliance with the License.
006 * You may obtain a copy of the License at
007 *
008 *     http://www.apache.org/licenses/LICENSE-2.0
009 *
010 * Unless required by applicable law or agreed to in writing, software
011 * distributed under the License is distributed on an "AS IS" BASIS,
012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 * See the License for the specific language governing permissions and
014 * limitations under the License.
015 *
016 * Contributors:
017 *     Tiago Cardoso <[email protected]>
018 */
019package org.nuxeo.ecm.platform.threed.convert;
020
021import static org.nuxeo.ecm.platform.threed.ThreeDConstants.SUPPORTED_EXTENSIONS;
022import static org.nuxeo.ecm.platform.threed.convert.Constants.BLENDER_PATH_PREFIX;
023import static org.nuxeo.ecm.platform.threed.convert.Constants.COORDS_PARAMETER;
024import static org.nuxeo.ecm.platform.threed.convert.Constants.DATA_PARAM;
025import static org.nuxeo.ecm.platform.threed.convert.Constants.DIMENSIONS_PARAMETER;
026import static org.nuxeo.ecm.platform.threed.convert.Constants.INPUT_FILE_PARAMETER;
027import static org.nuxeo.ecm.platform.threed.convert.Constants.LOD_IDS_PARAMETER;
028import static org.nuxeo.ecm.platform.threed.convert.Constants.MAX_POLY_PARAMETER;
029import static org.nuxeo.ecm.platform.threed.convert.Constants.NAME_PARAM;
030import static org.nuxeo.ecm.platform.threed.convert.Constants.OPERATORS_PARAMETER;
031import static org.nuxeo.ecm.platform.threed.convert.Constants.OUT_DIR_PARAMETER;
032import static org.nuxeo.ecm.platform.threed.convert.Constants.PERC_POLY_PARAMETER;
033import static org.nuxeo.ecm.platform.threed.convert.Constants.RENDER_IDS_PARAMETER;
034import static org.nuxeo.ecm.platform.threed.convert.Constants.SCRIPTS_DIRECTORY;
035import static org.nuxeo.ecm.platform.threed.convert.Constants.SCRIPTS_DIR_PARAMETER;
036import static org.nuxeo.ecm.platform.threed.convert.Constants.SCRIPTS_PIPELINE_DIRECTORY;
037import static org.nuxeo.ecm.platform.threed.convert.Constants.SCRIPT_FILE_PARAMETER;
038
039import java.io.Closeable;
040import java.io.File;
041import java.io.FileInputStream;
042import java.io.FileOutputStream;
043import java.io.IOException;
044import java.io.InputStream;
045import java.io.Serializable;
046import java.io.UncheckedIOException;
047import java.util.ArrayList;
048import java.util.Arrays;
049import java.util.Calendar;
050import java.util.Collections;
051import java.util.Comparator;
052import java.util.HashMap;
053import java.util.List;
054import java.util.Map;
055import java.util.UUID;
056import java.util.stream.Collectors;
057import java.util.stream.Stream;
058import java.util.zip.ZipEntry;
059import java.util.zip.ZipInputStream;
060
061import org.apache.commons.io.FileUtils;
062import org.apache.commons.io.FilenameUtils;
063import org.apache.commons.io.IOUtils;
064import org.nuxeo.common.Environment;
065import org.nuxeo.common.utils.Path;
066import org.nuxeo.ecm.core.api.Blob;
067import org.nuxeo.ecm.core.api.CloseableFile;
068import org.nuxeo.ecm.core.api.blobholder.BlobHolder;
069import org.nuxeo.ecm.core.api.blobholder.SimpleBlobHolderWithProperties;
070import org.nuxeo.ecm.core.api.impl.blob.FileBlob;
071import org.nuxeo.ecm.core.convert.api.ConversionException;
072import org.nuxeo.ecm.platform.commandline.executor.api.CmdParameters;
073import org.nuxeo.ecm.platform.commandline.executor.api.CommandException;
074import org.nuxeo.ecm.platform.commandline.executor.api.CommandLineExecutorService;
075import org.nuxeo.ecm.platform.commandline.executor.api.CommandNotAvailable;
076import org.nuxeo.ecm.platform.commandline.executor.api.ExecResult;
077import org.nuxeo.ecm.platform.convert.plugins.CommandLineBasedConverter;
078import org.nuxeo.runtime.api.Framework;
079
080/**
081 * Base converter for blender pipeline. Processes scripts for operators and input blobs
082 *
083 * @since 8.4
084 */
085public abstract class BaseBlenderConverter extends CommandLineBasedConverter {
086
087    public static final String MIMETYPE_ZIP = "application/zip";
088
089    protected Path tempDirectory(Map<String, Serializable> parameters, String sufix) throws ConversionException {
090        Path directory = new Path(getTmpDirectory(parameters)).append(BLENDER_PATH_PREFIX + UUID.randomUUID() + sufix);
091        boolean dirCreated = new File(directory.toString()).mkdirs();
092        if (!dirCreated) {
093            throw new ConversionException("Unable to create tmp dir: " + directory);
094        }
095        return directory;
096    }
097
098    protected boolean isThreeDFile(File file) {
099        return SUPPORTED_EXTENSIONS.contains(FilenameUtils.getExtension(file.getName()));
100    }
101
102    /**
103     * From a list of files, sort by file name and return a stream of file paths.
104     */
105    protected Stream<String> sortFilesToPaths(List<File> files) {
106        return files.stream().sorted(Comparator.comparing(File::getName)).map(File::getAbsolutePath);
107    }
108
109    private List<String> unpackZipFile(final File file, final File directory) throws IOException {
110        List<File> files3d = new ArrayList<>();
111        List<File> filesOther = new ArrayList<>();
112        ZipEntry zipEntry;
113        ZipInputStream zipInputStream = null;
114        try {
115            zipInputStream = new ZipInputStream(new FileInputStream(file));
116            while ((zipEntry = zipInputStream.getNextEntry()) != null) {
117                final File destFile = new File(directory, zipEntry.getName());
118                if (!zipEntry.isDirectory()) {
119                    try (FileOutputStream destOutputStream = new FileOutputStream(destFile)) {
120                        IOUtils.copy(zipInputStream, destOutputStream);
121                    }
122                    zipInputStream.closeEntry();
123                    if (destFile.isHidden()) {
124                        // ignore hidden files
125                        continue;
126                    }
127                    (isThreeDFile(destFile) ? files3d : filesOther).add(destFile);
128                } else {
129                    destFile.mkdirs();
130                }
131            }
132        } finally {
133            if (zipInputStream != null) {
134                zipInputStream.close();
135            }
136
137        }
138        // return a list of the sorted 3D files paths followed by the sorted non 3D files paths
139        return Stream.concat(sortFilesToPaths(files3d), sortFilesToPaths(filesOther)).collect(Collectors.toList());
140    }
141
142    protected List<String> blobsToTempDir(BlobHolder blobHolder, Path directory) throws IOException {
143        List<Blob> blobs = blobHolder.getBlobs();
144        List<String> filesCreated = new ArrayList<>();
145        if (blobs.isEmpty()) {
146            return filesCreated;
147        }
148
149        if (MIMETYPE_ZIP.equals(blobs.get(0).getMimeType())) {
150            filesCreated.addAll(unpackZipFile(blobs.get(0).getFile(), new File(directory.toString())));
151            blobs = blobs.subList(1, blobs.size());
152        }
153
154        // Add the main and assets as params
155        // The params are not used by the command but the blobs are extracted to files and managed!
156        filesCreated.addAll(blobs.stream().map(blob -> {
157            File file = new File(directory.append(blob.getFilename()).toString());
158            try {
159                blob.transferTo(file);
160            } catch (IOException e) {
161                throw new UncheckedIOException(e);
162            }
163            return file.getAbsolutePath();
164        }).collect(Collectors.toList()));
165
166        filesCreated.add(directory.toString());
167        return filesCreated;
168    }
169
170    private void createPath(Path path) throws ConversionException {
171        File pathFile = new File(path.toString());
172        if (!pathFile.exists()) {
173            boolean dirCreated = pathFile.mkdir();
174            if (!dirCreated) {
175                throw new ConversionException("Unable to create tmp dir for scripts output: " + path);
176            }
177        }
178    }
179
180    private Path copyScript(Path pathDst, Path source) throws IOException {
181        String sourceFile = source.lastSegment();
182        // xxx : find a way to check if the correct version is already there.
183        File script = new File(pathDst.append(sourceFile).toString());
184        InputStream is = getClass().getResourceAsStream("/" + source.toString());
185        FileUtils.copyInputStreamToFile(is, script);
186        return new Path(script.getAbsolutePath());
187    }
188
189    /**
190     * Returns the absolute path to the main script (main and pipeline scripts). Copies the script to the filesystem if
191     * needed. Copies needed {@code operators} script to the filesystem if missing.
192     */
193    protected Path getScriptWith(List<String> operators) throws IOException {
194        Path dataPath = new Path(Environment.getDefault().getData().getAbsolutePath());
195        String sourceDir = initParameters.get(SCRIPTS_DIR_PARAMETER);
196        String sourceFile = initParameters.get(SCRIPT_FILE_PARAMETER);
197
198        // create scripts directory
199        Path scriptsPath = dataPath.append(SCRIPTS_DIRECTORY);
200        createPath(scriptsPath);
201
202        copyScript(scriptsPath, new Path(sourceDir).append(sourceFile));
203
204        if (operators.isEmpty()) {
205            return scriptsPath;
206        }
207
208        // create pipeline scripts directory
209        Path pipelinePath = scriptsPath.append(SCRIPTS_PIPELINE_DIRECTORY);
210        createPath(pipelinePath);
211
212        Path pipelineSourcePath = new Path(sourceDir).append("pipeline");
213        // copy operators scripts resources
214        operators.forEach(operator -> {
215            try {
216                copyScript(pipelinePath, pipelineSourcePath.append(operator + ".py"));
217            } catch (IOException e) {
218                throw new UncheckedIOException(e);
219            }
220        });
221
222        return scriptsPath;
223    }
224
225    private List<String> getParams(Map<String, Serializable> inParams, Map<String, String> initParams, String key) {
226        String values = "";
227        if (inParams.containsKey(key)) {
228            values = (String) inParams.get(key);
229        } else if (initParams.containsKey(key)) {
230            values = initParams.get(key);
231        }
232        return Arrays.asList(values.split(" "));
233    }
234
235    @Override
236    public BlobHolder convert(BlobHolder blobHolder, Map<String, Serializable> parameters) throws ConversionException {
237        String dataContainer = "data" + String.valueOf(Calendar.getInstance().getTime().getTime());
238        String convertContainer = "convert" + String.valueOf(Calendar.getInstance().getTime().getTime());
239        String commandName = getCommandName(blobHolder, parameters);
240        if (commandName == null) {
241            throw new ConversionException("Unable to determine target CommandLine name");
242        }
243
244        List<Closeable> toClose = new ArrayList<>();
245        Path inDirectory = tempDirectory(null, "_in");
246        try {
247            CmdParameters params = new CmdParameters();
248
249            // Deal with operators and script files (blender and pipeline)
250            List<String> operatorsList = getParams(parameters, initParameters, OPERATORS_PARAMETER);
251            params.addNamedParameter(OPERATORS_PARAMETER, operatorsList);
252
253            operatorsList = operatorsList.stream().distinct().collect(Collectors.toList());
254            Path mainScriptDir = getScriptWith(operatorsList);
255            // params.addNamedParameter(SCRIPTS_DIR_PARAMETER, mainScriptDir.toString());
256            params.addNamedParameter(SCRIPT_FILE_PARAMETER, initParameters.get(SCRIPT_FILE_PARAMETER));
257
258            // Initialize render id params
259            params.addNamedParameter(RENDER_IDS_PARAMETER, getParams(parameters, initParameters, RENDER_IDS_PARAMETER));
260
261            // Initialize LOD id params
262            params.addNamedParameter(LOD_IDS_PARAMETER, getParams(parameters, initParameters, LOD_IDS_PARAMETER));
263
264            // Initialize percentage polygon params
265            params.addNamedParameter(PERC_POLY_PARAMETER, getParams(parameters, initParameters, PERC_POLY_PARAMETER));
266
267            // Initialize max polygon params
268            params.addNamedParameter(MAX_POLY_PARAMETER, getParams(parameters, initParameters, MAX_POLY_PARAMETER));
269
270            // Initialize spherical coordinates params
271            params.addNamedParameter(COORDS_PARAMETER, getParams(parameters, initParameters, COORDS_PARAMETER));
272
273            // Initialize dimension params
274            params.addNamedParameter(DIMENSIONS_PARAMETER, getParams(parameters, initParameters, DIMENSIONS_PARAMETER));
275
276            // Deal with input blobs (main and assets)
277            List<String> inputFiles = blobsToTempDir(blobHolder, inDirectory);
278            Path mainFile = new Path(inputFiles.get(0));
279            // params.addNamedParameter(INPUT_DIR_PARAMETER, mainFile.removeLastSegments(1).toString() );
280            params.addNamedParameter(INPUT_FILE_PARAMETER, mainFile.lastSegment());
281
282            // Extra blob parameters
283            Map<String, Blob> blobParams = getCmdBlobParameters(blobHolder, parameters);
284
285            ExecResult createRes = DockerHelper.CreateContainer(dataContainer, "nuxeo/blender");
286            if (createRes == null || !createRes.isSuccessful()) {
287                throw new ConversionException("Unable to create data volume : " + dataContainer,
288                        (createRes != null) ? createRes.getError() : null);
289            }
290            ExecResult copyRes = DockerHelper.CopyData(
291                    mainFile.removeLastSegments(1).toString() + File.separatorChar + ".", dataContainer + ":/in/");
292            if (copyRes == null || !copyRes.isSuccessful()) {
293                throw new ConversionException("Unable to copy content to data volume : " + dataContainer,
294                        (copyRes != null) ? copyRes.getError() : null);
295            }
296            copyRes = DockerHelper.CopyData(mainScriptDir.toString() + File.separatorChar + ".",
297                    dataContainer + ":/scripts/");
298            if (copyRes == null || !copyRes.isSuccessful()) {
299                throw new ConversionException("Unable to copy to scripts data volume : " + dataContainer,
300                        (copyRes != null) ? copyRes.getError() : null);
301            }
302            params.addNamedParameter(NAME_PARAM, convertContainer);
303            params.addNamedParameter(DATA_PARAM, dataContainer);
304
305            if (blobParams != null) {
306                for (String blobParamName : blobParams.keySet()) {
307                    Blob blob = blobParams.get(blobParamName);
308                    // closed in finally block
309                    CloseableFile closeable = blob.getCloseableFile(
310                            "." + FilenameUtils.getExtension(blob.getFilename()));
311                    params.addNamedParameter(blobParamName, closeable.getFile());
312                    toClose.add(closeable);
313                }
314            }
315
316            // Extra string parameters
317            Map<String, String> strParams = getCmdStringParameters(blobHolder, parameters);
318
319            if (strParams != null) {
320                for (String paramName : strParams.keySet()) {
321                    if (RENDER_IDS_PARAMETER.equals(paramName) || LOD_IDS_PARAMETER.equals(paramName)
322                            || PERC_POLY_PARAMETER.equals(paramName) || MAX_POLY_PARAMETER.equals(paramName)
323                            || COORDS_PARAMETER.equals(paramName) || DIMENSIONS_PARAMETER.equals(paramName)) {
324                        if (strParams.get(paramName) != null) {
325                            params.addNamedParameter(paramName, Arrays.asList(strParams.get(paramName).split(" ")));
326                        }
327                    } else {
328                        params.addNamedParameter(paramName, strParams.get(paramName));
329                    }
330                }
331            }
332
333            // Deal with output directory
334            Path outDir = tempDirectory(null, "_out");
335            params.addNamedParameter(OUT_DIR_PARAMETER, outDir.toString());
336
337            ExecResult result = Framework.getService(CommandLineExecutorService.class).execCommand(commandName, params);
338            if (!result.isSuccessful()) {
339                throw result.getError();
340            }
341
342            copyRes = DockerHelper.CopyData(dataContainer + ":/out/.", outDir.toString());
343            if (copyRes == null || !copyRes.isSuccessful()) {
344                throw new ConversionException("Unable to copy from data volume : " + dataContainer,
345                        (copyRes != null) ? copyRes.getError() : null);
346            }
347            return buildResult(result.getOutput(), params);
348        } catch (CommandNotAvailable e) {
349            // XXX bubble installation instructions
350            throw new ConversionException("Unable to find targetCommand", e);
351        } catch (IOException | CommandException e) {
352            throw new ConversionException("Error while converting via CommandLineService", e);
353        } finally {
354            FileUtils.deleteQuietly(new File(inDirectory.toString()));
355            for (Closeable closeable : toClose) {
356                IOUtils.closeQuietly(closeable);
357            }
358            DockerHelper.RemoveContainer(dataContainer);
359            DockerHelper.RemoveContainer(convertContainer);
360        }
361
362    }
363
364    public List<String> getConversionLOD(String outDir) {
365        File directory = new File(outDir);
366        String[] files = directory.list((dir, name) -> true);
367        return (files == null) ? Collections.emptyList() : Arrays.asList(files);
368    }
369
370    public List<String> getRenders(String outDir) {
371        File directory = new File(outDir);
372        String[] files = directory.list((dir, name) -> name.startsWith("render") && name.endsWith(".png"));
373        return (files == null) ? Collections.emptyList() : Arrays.asList(files);
374    }
375
376    public List<String> getInfos(String outDir) {
377        File directory = new File(outDir);
378        String[] files = directory.list((dir, name) -> name.endsWith(".info"));
379        return (files == null) ? Collections.emptyList() : Arrays.asList(files);
380    }
381
382    @Override
383    protected BlobHolder buildResult(List<String> cmdOutput, CmdParameters cmdParams) throws ConversionException {
384        String outDir = cmdParams.getParameter(OUT_DIR_PARAMETER);
385        Map<String, Integer> lodBlobIndexes = new HashMap<>();
386        List<Integer> resourceIndexes = new ArrayList<>();
387        Map<String, Integer> infoIndexes = new HashMap<>();
388        List<Blob> blobs = new ArrayList<>();
389
390        String lodDir = outDir + File.separatorChar + "convert";
391        List<String> conversions = getConversionLOD(lodDir);
392        conversions.forEach(filename -> {
393            File file = new File(lodDir + File.separatorChar + filename);
394            Blob blob = new FileBlob(file);
395            blob.setFilename(file.getName());
396            if (FilenameUtils.getExtension(filename).toLowerCase().equals("dae")) {
397                String[] filenameArray = filename.split("-");
398                if (filenameArray.length != 4) {
399                    throw new ConversionException(
400                            Arrays.toString(filenameArray) + " incompatible with conversion file name schema.");
401                }
402                lodBlobIndexes.put(filenameArray[1], blobs.size());
403            } else {
404                resourceIndexes.add(blobs.size());
405            }
406            blobs.add(blob);
407        });
408
409        String infoDir = outDir + File.separatorChar + "info";
410        List<String> infos = getInfos(infoDir);
411        infos.forEach(filename -> {
412            File file = new File(infoDir + File.separatorChar + filename);
413            Blob blob = new FileBlob(file);
414            blob.setFilename(file.getName());
415            if (FilenameUtils.getExtension(filename).toLowerCase().equals("info")) {
416                String lodId = FilenameUtils.getBaseName(filename);
417                infoIndexes.put(lodId, blobs.size());
418                blobs.add(blob);
419            }
420        });
421
422        String renderDir = outDir + File.separatorChar + "render";
423        List<String> renders = getRenders(renderDir);
424
425        Map<String, Serializable> properties = new HashMap<>();
426        properties.put("cmdOutput", (Serializable) cmdOutput);
427        properties.put("resourceIndexes", (Serializable) resourceIndexes);
428        properties.put("infoIndexes", (Serializable) infoIndexes);
429        properties.put("lodIdIndexes", (Serializable) lodBlobIndexes);
430        properties.put("renderStartIndex", blobs.size());
431
432        blobs.addAll(renders.stream().map(result -> {
433            File file = new File(renderDir + File.separatorChar + result);
434            Blob blob = new FileBlob(file);
435            blob.setFilename(file.getName());
436            return blob;
437        }).collect(Collectors.toList()));
438
439        return new SimpleBlobHolderWithProperties(blobs, properties);
440    }
441}