").append(x.parseHTML(e)).find(r):e)}).complete(n&&function(e,t){s.each(n,o||[e.responseText,t,e])}),this},x.each(["ajaxStart","ajaxStop","ajaxComplete","ajaxError","ajaxSuccess","ajaxSend"],function(e,t){x.fn[t]=function(e){return this.on(t,e)}}),x.extend({active:0,lastModified:{},etag:{},ajaxSettings:{url:Ut,type:"GET",isLocal:Kt.test(Xt[1]),global:!0,processData:!0,async:!0,contentType:"application/x-www-form-urlencoded; charset=UTF-8",accepts:{"*":sn,text:"text/plain",html:"text/html",xml:"application/xml, text/xml",json:"application/json, text/javascript"},contents:{xml:/xml/,html:/html/,json:/json/},responseFields:{xml:"responseXML",text:"responseText",json:"responseJSON"},converters:{"* text":String,"text html":!0,"text json":x.parseJSON,"text xml":x.parseXML},flatOptions:{url:!0,context:!0}},ajaxSetup:function(e,t){return t?cn(cn(e,x.ajaxSettings),t):cn(x.ajaxSettings,e)},ajaxPrefilter:un(rn),ajaxTransport:un(on),ajax:function(e,t){"object"==typeof e&&(t=e,e=undefined),t=t||{};var n,r,i,o,s,a,u,l,c=x.ajaxSetup({},t),p=c.context||c,f=c.context&&(p.nodeType||p.jquery)?x(p):x.event,h=x.Deferred(),d=x.Callbacks("once memory"),g=c.statusCode||{},m={},y={},v=0,b="canceled",T={readyState:0,getResponseHeader:function(e){var t;if(2===v){if(!o){o={};while(t=Qt.exec(i))o[t[1].toLowerCase()]=t[2]}t=o[e.toLowerCase()]}return null==t?null:t},getAllResponseHeaders:function(){return 2===v?i:null},setRequestHeader:function(e,t){var n=e.toLowerCase();return v||(e=y[n]=y[n]||e,m[e]=t),this},overrideMimeType:function(e){return v||(c.mimeType=e),this},statusCode:function(e){var t;if(e)if(2>v)for(t in e)g[t]=[g[t],e[t]];else T.always(e[T.status]);return this},abort:function(e){var t=e||b;return n&&n.abort(t),k(0,t),this}};if(h.promise(T).complete=d.add,T.success=T.done,T.error=T.fail,c.url=((e||c.url||Ut)+"").replace(Gt,"").replace(en,Xt[1]+"//"),c.type=t.method||t.type||c.method||c.type,c.dataTypes=x.trim(c.dataType||"*").toLowerCase().match(w)||[""],null==c.crossDomain&&(a=tn.exec(c.url.toLowerCase()),c.crossDomain=!(!a||a[1]===Xt[1]&&a[2]===Xt[2]&&(a[3]||("http:"===a[1]?"80":"443"))===(Xt[3]||("http:"===Xt[1]?"80":"443")))),c.data&&c.processData&&"string"!=typeof c.data&&(c.data=x.param(c.data,c.traditional)),ln(rn,c,t,T),2===v)return T;u=c.global,u&&0===x.active++&&x.event.trigger("ajaxStart"),c.type=c.type.toUpperCase(),c.hasContent=!Zt.test(c.type),r=c.url,c.hasContent||(c.data&&(r=c.url+=(Vt.test(r)?"&":"?")+c.data,delete c.data),c.cache===!1&&(c.url=Jt.test(r)?r.replace(Jt,"$1_="+Yt++):r+(Vt.test(r)?"&":"?")+"_="+Yt++)),c.ifModified&&(x.lastModified[r]&&T.setRequestHeader("If-Modified-Since",x.lastModified[r]),x.etag[r]&&T.setRequestHeader("If-None-Match",x.etag[r])),(c.data&&c.hasContent&&c.contentType!==!1||t.contentType)&&T.setRequestHeader("Content-Type",c.contentType),T.setRequestHeader("Accept",c.dataTypes[0]&&c.accepts[c.dataTypes[0]]?c.accepts[c.dataTypes[0]]+("*"!==c.dataTypes[0]?", "+sn+"; q=0.01":""):c.accepts["*"]);for(l in c.headers)T.setRequestHeader(l,c.headers[l]);if(c.beforeSend&&(c.beforeSend.call(p,T,c)===!1||2===v))return T.abort();b="abort";for(l in{success:1,error:1,complete:1})T[l](c[l]);if(n=ln(on,c,t,T)){T.readyState=1,u&&f.trigger("ajaxSend",[T,c]),c.async&&c.timeout>0&&(s=setTimeout(function(){T.abort("timeout")},c.timeout));try{v=1,n.send(m,k)}catch(C){if(!(2>v))throw C;k(-1,C)}}else k(-1,"No Transport");function k(e,t,o,a){var l,m,y,b,w,C=t;2!==v&&(v=2,s&&clearTimeout(s),n=undefined,i=a||"",T.readyState=e>0?4:0,l=e>=200&&300>e||304===e,o&&(b=pn(c,T,o)),b=fn(c,b,T,l),l?(c.ifModified&&(w=T.getResponseHeader("Last-Modified"),w&&(x.lastModified[r]=w),w=T.getResponseHeader("etag"),w&&(x.etag[r]=w)),204===e||"HEAD"===c.type?C="nocontent":304===e?C="notmodified":(C=b.state,m=b.data,y=b.error,l=!y)):(y=C,(e||!C)&&(C="error",0>e&&(e=0))),T.status=e,T.statusText=(t||C)+"",l?h.resolveWith(p,[m,C,T]):h.rejectWith(p,[T,C,y]),T.statusCode(g),g=undefined,u&&f.trigger(l?"ajaxSuccess":"ajaxError",[T,c,l?m:y]),d.fireWith(p,[T,C]),u&&(f.trigger("ajaxComplete",[T,c]),--x.active||x.event.trigger("ajaxStop")))}return T},getJSON:function(e,t,n){return x.get(e,t,n,"json")},getScript:function(e,t){return x.get(e,undefined,t,"script")}}),x.each(["get","post"],function(e,t){x[t]=function(e,n,r,i){return x.isFunction(n)&&(i=i||r,r=n,n=undefined),x.ajax({url:e,type:t,dataType:i,data:n,success:r})}});function pn(e,t,n){var r,i,o,s,a=e.contents,u=e.dataTypes;while("*"===u[0])u.shift(),r===undefined&&(r=e.mimeType||t.getResponseHeader("Content-Type"));if(r)for(i in a)if(a[i]&&a[i].test(r)){u.unshift(i);break}if(u[0]in n)o=u[0];else{for(i in n){if(!u[0]||e.converters[i+" "+u[0]]){o=i;break}s||(s=i)}o=o||s}return o?(o!==u[0]&&u.unshift(o),n[o]):undefined}function fn(e,t,n,r){var i,o,s,a,u,l={},c=e.dataTypes.slice();if(c[1])for(s in e.converters)l[s.toLowerCase()]=e.converters[s];o=c.shift();while(o)if(e.responseFields[o]&&(n[e.responseFields[o]]=t),!u&&r&&e.dataFilter&&(t=e.dataFilter(t,e.dataType)),u=o,o=c.shift())if("*"===o)o=u;else if("*"!==u&&u!==o){if(s=l[u+" "+o]||l["* "+o],!s)for(i in l)if(a=i.split(" "),a[1]===o&&(s=l[u+" "+a[0]]||l["* "+a[0]])){s===!0?s=l[i]:l[i]!==!0&&(o=a[0],c.unshift(a[1]));break}if(s!==!0)if(s&&e["throws"])t=s(t);else try{t=s(t)}catch(p){return{state:"parsererror",error:s?p:"No conversion from "+u+" to "+o}}}return{state:"success",data:t}}x.ajaxSetup({accepts:{script:"text/javascript, application/javascript, application/ecmascript, application/x-ecmascript"},contents:{script:/(?:java|ecma)script/},converters:{"text script":function(e){return x.globalEval(e),e}}}),x.ajaxPrefilter("script",function(e){e.cache===undefined&&(e.cache=!1),e.crossDomain&&(e.type="GET")}),x.ajaxTransport("script",function(e){if(e.crossDomain){var t,n;return{send:function(r,i){t=x("
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
Workfiles...
+
Create...
+
Load...
+
Publish...
+
Manage...
+
Subset Manager...
+
Experimental Tools...
+
+
diff --git a/openpype/hosts/photoshop/api/launch_logic.py b/openpype/hosts/photoshop/api/launch_logic.py
new file mode 100644
index 00000000000..0bbb19523d0
--- /dev/null
+++ b/openpype/hosts/photoshop/api/launch_logic.py
@@ -0,0 +1,364 @@
+import os
+import subprocess
+import collections
+import asyncio
+
+from wsrpc_aiohttp import (
+ WebSocketRoute,
+ WebSocketAsync
+)
+
+from Qt import QtCore
+
+from openpype.api import Logger
+from openpype.pipeline import legacy_io
+from openpype.tools.utils import host_tools
+from openpype.tools.adobe_webserver.app import WebServerTool
+
+from .ws_stub import PhotoshopServerStub
+
+log = Logger.get_logger(__name__)
+
+
+class ConnectionNotEstablishedYet(Exception):
+ pass
+
+
+class MainThreadItem:
+ """Structure to store information about callback in main thread.
+
+ Item should be used to execute callback in main thread which may be needed
+ for execution of Qt objects.
+
+ Item store callback (callable variable), arguments and keyword arguments
+ for the callback. Item hold information about it's process.
+ """
+ not_set = object()
+
+ def __init__(self, callback, *args, **kwargs):
+ self._done = False
+ self._exception = self.not_set
+ self._result = self.not_set
+ self._callback = callback
+ self._args = args
+ self._kwargs = kwargs
+
+ @property
+ def done(self):
+ return self._done
+
+ @property
+ def exception(self):
+ return self._exception
+
+ @property
+ def result(self):
+ return self._result
+
+ def execute(self):
+ """Execute callback and store it's result.
+
+ Method must be called from main thread. Item is marked as `done`
+ when callback execution finished. Store output of callback of exception
+ information when callback raise one.
+ """
+ log.debug("Executing process in main thread")
+ if self.done:
+ log.warning("- item is already processed")
+ return
+
+ log.info("Running callback: {}".format(str(self._callback)))
+ try:
+ result = self._callback(*self._args, **self._kwargs)
+ self._result = result
+
+ except Exception as exc:
+ self._exception = exc
+
+ finally:
+ self._done = True
+
+
+def stub():
+ """
+ Convenience function to get server RPC stub to call methods directed
+ for host (Photoshop).
+ It expects already created connection, started from client.
+ Currently created when panel is opened (PS: Window>Extensions>Avalon)
+ :return:
where functions could be called from
+ """
+ ps_stub = PhotoshopServerStub()
+ if not ps_stub.client:
+ raise ConnectionNotEstablishedYet("Connection is not created yet")
+
+ return ps_stub
+
+
+def show_tool_by_name(tool_name):
+ kwargs = {}
+ if tool_name == "loader":
+ kwargs["use_context"] = True
+
+ host_tools.show_tool_by_name(tool_name, **kwargs)
+
+
+class ProcessLauncher(QtCore.QObject):
+ route_name = "Photoshop"
+ _main_thread_callbacks = collections.deque()
+
+ def __init__(self, subprocess_args):
+ self._subprocess_args = subprocess_args
+ self._log = None
+
+ super(ProcessLauncher, self).__init__()
+
+ # Keep track if launcher was already started
+ self._started = False
+
+ self._process = None
+ self._websocket_server = None
+
+ start_process_timer = QtCore.QTimer()
+ start_process_timer.setInterval(100)
+
+ loop_timer = QtCore.QTimer()
+ loop_timer.setInterval(200)
+
+ start_process_timer.timeout.connect(self._on_start_process_timer)
+ loop_timer.timeout.connect(self._on_loop_timer)
+
+ self._start_process_timer = start_process_timer
+ self._loop_timer = loop_timer
+
+ @property
+ def log(self):
+ if self._log is None:
+ self._log = Logger.get_logger(
+ "{}-launcher".format(self.route_name)
+ )
+ return self._log
+
+ @property
+ def websocket_server_is_running(self):
+ if self._websocket_server is not None:
+ return self._websocket_server.is_running
+ return False
+
+ @property
+ def is_process_running(self):
+ if self._process is not None:
+ return self._process.poll() is None
+ return False
+
+ @property
+ def is_host_connected(self):
+ """Returns True if connected, False if app is not running at all."""
+ if not self.is_process_running:
+ return False
+
+ try:
+ _stub = stub()
+ if _stub:
+ return True
+ except Exception:
+ pass
+
+ return None
+
+ @classmethod
+ def execute_in_main_thread(cls, callback, *args, **kwargs):
+ item = MainThreadItem(callback, *args, **kwargs)
+ cls._main_thread_callbacks.append(item)
+ return item
+
+ def start(self):
+ if self._started:
+ return
+ self.log.info("Started launch logic of Photoshop")
+ self._started = True
+ self._start_process_timer.start()
+
+ def exit(self):
+ """ Exit whole application. """
+ if self._start_process_timer.isActive():
+ self._start_process_timer.stop()
+ if self._loop_timer.isActive():
+ self._loop_timer.stop()
+
+ if self._websocket_server is not None:
+ self._websocket_server.stop()
+
+ if self._process:
+ self._process.kill()
+ self._process.wait()
+
+ QtCore.QCoreApplication.exit()
+
+ def _on_loop_timer(self):
+ # TODO find better way and catch errors
+ # Run only callbacks that are in queue at the moment
+ cls = self.__class__
+ for _ in range(len(cls._main_thread_callbacks)):
+ if cls._main_thread_callbacks:
+ item = cls._main_thread_callbacks.popleft()
+ item.execute()
+
+ if not self.is_process_running:
+ self.log.info("Host process is not running. Closing")
+ self.exit()
+
+ elif not self.websocket_server_is_running:
+ self.log.info("Websocket server is not running. Closing")
+ self.exit()
+
+ def _on_start_process_timer(self):
+ # TODO add try except validations for each part in this method
+ # Start server as first thing
+ if self._websocket_server is None:
+ self._init_server()
+ return
+
+ # TODO add waiting time
+ # Wait for webserver
+ if not self.websocket_server_is_running:
+ return
+
+ # Start application process
+ if self._process is None:
+ self._start_process()
+ self.log.info("Waiting for host to connect")
+ return
+
+ # TODO add waiting time
+ # Wait until host is connected
+ if self.is_host_connected:
+ self._start_process_timer.stop()
+ self._loop_timer.start()
+ elif (
+ not self.is_process_running
+ or not self.websocket_server_is_running
+ ):
+ self.exit()
+
+ def _init_server(self):
+ if self._websocket_server is not None:
+ return
+
+ self.log.debug(
+ "Initialization of websocket server for host communication"
+ )
+
+ self._websocket_server = websocket_server = WebServerTool()
+ if websocket_server.port_occupied(
+ websocket_server.host_name,
+ websocket_server.port
+ ):
+ self.log.info(
+ "Server already running, sending actual context and exit."
+ )
+ asyncio.run(websocket_server.send_context_change(self.route_name))
+ self.exit()
+ return
+
+ # Add Websocket route
+ websocket_server.add_route("*", "/ws/", WebSocketAsync)
+ # Add after effects route to websocket handler
+
+ print("Adding {} route".format(self.route_name))
+ WebSocketAsync.add_route(
+ self.route_name, PhotoshopRoute
+ )
+ self.log.info("Starting websocket server for host communication")
+ websocket_server.start_server()
+
+ def _start_process(self):
+ if self._process is not None:
+ return
+ self.log.info("Starting host process")
+ try:
+ self._process = subprocess.Popen(
+ self._subprocess_args,
+ stdout=subprocess.DEVNULL,
+ stderr=subprocess.DEVNULL
+ )
+ except Exception:
+ self.log.info("exce", exc_info=True)
+ self.exit()
+
+
+class PhotoshopRoute(WebSocketRoute):
+ """
+ One route, mimicking external application (like Harmony, etc).
+ All functions could be called from client.
+ 'do_notify' function calls function on the client - mimicking
+ notification after long running job on the server or similar
+ """
+ instance = None
+
+ def init(self, **kwargs):
+ # Python __init__ must be return "self".
+ # This method might return anything.
+ log.debug("someone called Photoshop route")
+ self.instance = self
+ return kwargs
+
+ # server functions
+ async def ping(self):
+ log.debug("someone called Photoshop route ping")
+
+ # This method calls function on the client side
+ # client functions
+ async def set_context(self, project, asset, task):
+ """
+ Sets 'project' and 'asset' to envs, eg. setting context
+
+ Args:
+ project (str)
+ asset (str)
+ """
+ log.info("Setting context change")
+ log.info("project {} asset {} ".format(project, asset))
+ if project:
+ legacy_io.Session["AVALON_PROJECT"] = project
+ os.environ["AVALON_PROJECT"] = project
+ if asset:
+ legacy_io.Session["AVALON_ASSET"] = asset
+ os.environ["AVALON_ASSET"] = asset
+ if task:
+ legacy_io.Session["AVALON_TASK"] = task
+ os.environ["AVALON_TASK"] = task
+
+ async def read(self):
+ log.debug("photoshop.read client calls server server calls "
+ "photoshop client")
+ return await self.socket.call('photoshop.read')
+
+ # panel routes for tools
+ async def creator_route(self):
+ self._tool_route("creator")
+
+ async def workfiles_route(self):
+ self._tool_route("workfiles")
+
+ async def loader_route(self):
+ self._tool_route("loader")
+
+ async def publish_route(self):
+ self._tool_route("publish")
+
+ async def sceneinventory_route(self):
+ self._tool_route("sceneinventory")
+
+ async def subsetmanager_route(self):
+ self._tool_route("subsetmanager")
+
+ async def experimental_tools_route(self):
+ self._tool_route("experimental_tools")
+
+ def _tool_route(self, _tool_name):
+ """The address accessed when clicking on the buttons."""
+
+ ProcessLauncher.execute_in_main_thread(show_tool_by_name, _tool_name)
+
+ # Required return statement.
+ return "nothing"
diff --git a/openpype/hosts/photoshop/api/lib.py b/openpype/hosts/photoshop/api/lib.py
new file mode 100644
index 00000000000..221b4314e6f
--- /dev/null
+++ b/openpype/hosts/photoshop/api/lib.py
@@ -0,0 +1,83 @@
+import os
+import sys
+import contextlib
+import traceback
+
+from Qt import QtWidgets
+
+from openpype.lib import env_value_to_bool, Logger
+from openpype.modules import ModulesManager
+from openpype.pipeline import install_host
+from openpype.tools.utils import host_tools
+
+from .launch_logic import ProcessLauncher, stub
+
+log = Logger.get_logger(__name__)
+
+
+def safe_excepthook(*args):
+ traceback.print_exception(*args)
+
+
+def main(*subprocess_args):
+ from openpype.hosts.photoshop import api
+
+ install_host(api)
+ sys.excepthook = safe_excepthook
+
+ # coloring in StdOutBroker
+ os.environ["OPENPYPE_LOG_NO_COLORS"] = "False"
+ app = QtWidgets.QApplication([])
+ app.setQuitOnLastWindowClosed(False)
+
+ launcher = ProcessLauncher(subprocess_args)
+ launcher.start()
+
+ if env_value_to_bool("HEADLESS_PUBLISH"):
+ manager = ModulesManager()
+ webpublisher_addon = manager["webpublisher"]
+ launcher.execute_in_main_thread(
+ webpublisher_addon.headless_publish,
+ log,
+ "ClosePS",
+ os.environ.get("IS_TEST")
+ )
+ elif env_value_to_bool("AVALON_PHOTOSHOP_WORKFILES_ON_LAUNCH",
+ default=True):
+
+ launcher.execute_in_main_thread(
+ host_tools.show_workfiles,
+ save=env_value_to_bool("WORKFILES_SAVE_AS")
+ )
+
+ sys.exit(app.exec_())
+
+
+@contextlib.contextmanager
+def maintained_selection():
+ """Maintain selection during context."""
+ selection = stub().get_selected_layers()
+ try:
+ yield selection
+ finally:
+ stub().select_layers(selection)
+
+
+@contextlib.contextmanager
+def maintained_visibility(layers=None):
+ """Maintain visibility during context.
+
+ Args:
+ layers (list) of PSItem (used for caching)
+ """
+ visibility = {}
+ if not layers:
+ layers = stub().get_layers()
+ for layer in layers:
+ visibility[layer.id] = layer.visible
+ try:
+ yield
+ finally:
+ for layer in layers:
+ stub().set_visible(layer.id, visibility[layer.id])
+ pass
diff --git a/openpype/hosts/photoshop/api/panel.PNG b/openpype/hosts/photoshop/api/panel.PNG
new file mode 100644
index 00000000000..be5db3b8df0
Binary files /dev/null and b/openpype/hosts/photoshop/api/panel.PNG differ
diff --git a/openpype/hosts/photoshop/api/panel_failure.PNG b/openpype/hosts/photoshop/api/panel_failure.PNG
new file mode 100644
index 00000000000..67afc4e2125
Binary files /dev/null and b/openpype/hosts/photoshop/api/panel_failure.PNG differ
diff --git a/openpype/hosts/photoshop/api/pipeline.py b/openpype/hosts/photoshop/api/pipeline.py
new file mode 100644
index 00000000000..f6600966306
--- /dev/null
+++ b/openpype/hosts/photoshop/api/pipeline.py
@@ -0,0 +1,254 @@
+import os
+from Qt import QtWidgets
+
+import pyblish.api
+
+from openpype.api import Logger
+from openpype.lib import register_event_callback
+from openpype.pipeline import (
+ legacy_io,
+ register_loader_plugin_path,
+ register_creator_plugin_path,
+ deregister_loader_plugin_path,
+ deregister_creator_plugin_path,
+ AVALON_CONTAINER_ID,
+)
+from openpype.pipeline.load import any_outdated_containers
+from openpype.hosts.photoshop import PHOTOSHOP_HOST_DIR
+
+from . import lib
+
+log = Logger.get_logger(__name__)
+
+PLUGINS_DIR = os.path.join(PHOTOSHOP_HOST_DIR, "plugins")
+PUBLISH_PATH = os.path.join(PLUGINS_DIR, "publish")
+LOAD_PATH = os.path.join(PLUGINS_DIR, "load")
+CREATE_PATH = os.path.join(PLUGINS_DIR, "create")
+INVENTORY_PATH = os.path.join(PLUGINS_DIR, "inventory")
+
+
+def check_inventory():
+ if not any_outdated_containers():
+ return
+
+ # Warn about outdated containers.
+ _app = QtWidgets.QApplication.instance()
+ if not _app:
+ print("Starting new QApplication..")
+ _app = QtWidgets.QApplication([])
+
+ message_box = QtWidgets.QMessageBox()
+ message_box.setIcon(QtWidgets.QMessageBox.Warning)
+ msg = "There are outdated containers in the scene."
+ message_box.setText(msg)
+ message_box.exec_()
+
+
+def on_application_launch():
+ check_inventory()
+
+
+def on_pyblish_instance_toggled(instance, old_value, new_value):
+ """Toggle layer visibility on instance toggles."""
+ instance[0].Visible = new_value
+
+
+def install():
+ """Install Photoshop-specific functionality of avalon-core.
+
+ This function is called automatically on calling `api.install(photoshop)`.
+ """
+ log.info("Installing OpenPype Photoshop...")
+ pyblish.api.register_host("photoshop")
+
+ pyblish.api.register_plugin_path(PUBLISH_PATH)
+ register_loader_plugin_path(LOAD_PATH)
+ register_creator_plugin_path(CREATE_PATH)
+ log.info(PUBLISH_PATH)
+
+ pyblish.api.register_callback(
+ "instanceToggled", on_pyblish_instance_toggled
+ )
+
+ register_event_callback("application.launched", on_application_launch)
+
+
+def uninstall():
+ pyblish.api.deregister_plugin_path(PUBLISH_PATH)
+ deregister_loader_plugin_path(LOAD_PATH)
+ deregister_creator_plugin_path(CREATE_PATH)
+
+
+def ls():
+ """Yields containers from active Photoshop document
+
+ This is the host-equivalent of api.ls(), but instead of listing
+ assets on disk, it lists assets already loaded in Photoshop; once loaded
+ they are called 'containers'
+
+ Yields:
+ dict: container
+
+ """
+ try:
+ stub = lib.stub() # only after Photoshop is up
+ except lib.ConnectionNotEstablishedYet:
+ print("Not connected yet, ignoring")
+ return
+
+ if not stub.get_active_document_name():
+ return
+
+ layers_meta = stub.get_layers_metadata() # minimalize calls to PS
+ for layer in stub.get_layers():
+ data = stub.read(layer, layers_meta)
+
+ # Skip non-tagged layers.
+ if not data:
+ continue
+
+ # Filter to only containers.
+ if "container" not in data["id"]:
+ continue
+
+ # Append transient data
+ data["objectName"] = layer.name.replace(stub.LOADED_ICON, '')
+ data["layer"] = layer
+
+ yield data
+
+
+def list_instances():
+ """List all created instances to publish from current workfile.
+
+ Pulls from File > File Info
+
+ For SubsetManager
+
+ Returns:
+ (list) of dictionaries matching instances format
+ """
+ stub = _get_stub()
+
+ if not stub:
+ return []
+
+ instances = []
+ layers_meta = stub.get_layers_metadata()
+ if layers_meta:
+ for instance in layers_meta:
+ if instance.get("id") == "pyblish.avalon.instance":
+ instances.append(instance)
+
+ return instances
+
+
+def remove_instance(instance):
+ """Remove instance from current workfile metadata.
+
+ Updates metadata of current file in File > File Info and removes
+ icon highlight on group layer.
+
+ For SubsetManager
+
+ Args:
+ instance (dict): instance representation from subsetmanager model
+ """
+ stub = _get_stub()
+
+ if not stub:
+ return
+
+ inst_id = instance.get("instance_id") or instance.get("uuid") # legacy
+ if not inst_id:
+ log.warning("No instance identifier for {}".format(instance))
+ return
+
+ stub.remove_instance(inst_id)
+
+ if instance.get("members"):
+ item = stub.get_layer(instance["members"][0])
+ if item:
+ stub.rename_layer(item.id,
+ item.name.replace(stub.PUBLISH_ICON, ''))
+
+
+def _get_stub():
+ """Handle pulling stub from PS to run operations on host
+
+ Returns:
+ (PhotoshopServerStub) or None
+ """
+ try:
+ stub = lib.stub() # only after Photoshop is up
+ except lib.ConnectionNotEstablishedYet:
+ print("Not connected yet, ignoring")
+ return
+
+ if not stub.get_active_document_name():
+ return
+
+ return stub
+
+
+def containerise(
+ name, namespace, layer, context, loader=None, suffix="_CON"
+):
+ """Imprint layer with metadata
+
+ Containerisation enables a tracking of version, author and origin
+ for loaded assets.
+
+ Arguments:
+ name (str): Name of resulting assembly
+ namespace (str): Namespace under which to host container
+ layer (PSItem): Layer to containerise
+ context (dict): Asset information
+ loader (str, optional): Name of loader used to produce this container.
+ suffix (str, optional): Suffix of container, defaults to `_CON`.
+
+ Returns:
+ container (str): Name of container assembly
+ """
+ layer.name = name + suffix
+
+ data = {
+ "schema": "openpype:container-2.0",
+ "id": AVALON_CONTAINER_ID,
+ "name": name,
+ "namespace": namespace,
+ "loader": str(loader),
+ "representation": str(context["representation"]["_id"]),
+ "members": [str(layer.id)]
+ }
+ stub = lib.stub()
+ stub.imprint(layer.id, data)
+
+ return layer
+
+
+def get_context_data():
+ """Get stored values for context (validation enable/disable etc)"""
+ meta = _get_stub().get_layers_metadata()
+ for item in meta:
+ if item.get("id") == "publish_context":
+ item.pop("id")
+ return item
+
+ return {}
+
+
+def update_context_data(data, changes):
+ """Store value needed for context"""
+ item = data
+ item["id"] = "publish_context"
+ _get_stub().imprint(item["id"], item)
+
+
+def get_context_title():
+ """Returns title for Creator window"""
+
+ project_name = legacy_io.Session["AVALON_PROJECT"]
+ asset_name = legacy_io.Session["AVALON_ASSET"]
+ task_name = legacy_io.Session["AVALON_TASK"]
+ return "{}/{}/{}".format(project_name, asset_name, task_name)
diff --git a/openpype/hosts/photoshop/api/plugin.py b/openpype/hosts/photoshop/api/plugin.py
new file mode 100644
index 00000000000..c80e6bbd063
--- /dev/null
+++ b/openpype/hosts/photoshop/api/plugin.py
@@ -0,0 +1,35 @@
+import re
+
+from openpype.pipeline import LoaderPlugin
+from .launch_logic import stub
+
+
+def get_unique_layer_name(layers, asset_name, subset_name):
+ """
+ Gets all layer names and if 'asset_name_subset_name' is present, it
+ increases suffix by 1 (eg. creates unique layer name - for Loader)
+ Args:
+ layers (list) of dict with layers info (name, id etc.)
+ asset_name (string):
+ subset_name (string):
+
+ Returns:
+ (string): name_00X (without version)
+ """
+ name = "{}_{}".format(asset_name, subset_name)
+ names = {}
+ for layer in layers:
+ layer_name = re.sub(r'_\d{3}$', '', layer.name)
+ if layer_name in names.keys():
+ names[layer_name] = names[layer_name] + 1
+ else:
+ names[layer_name] = 1
+ occurrences = names.get(name, 0)
+
+ return "{}_{:0>3d}".format(name, occurrences + 1)
+
+
+class PhotoshopLoader(LoaderPlugin):
+ @staticmethod
+ def get_stub():
+ return stub()
diff --git a/openpype/hosts/photoshop/api/workio.py b/openpype/hosts/photoshop/api/workio.py
new file mode 100644
index 00000000000..35b44d60708
--- /dev/null
+++ b/openpype/hosts/photoshop/api/workio.py
@@ -0,0 +1,49 @@
+"""Host API required Work Files tool"""
+import os
+
+from . import lib
+
+
+def _active_document():
+ document_name = lib.stub().get_active_document_name()
+ if not document_name:
+ return None
+
+ return document_name
+
+
+def file_extensions():
+ return [".psd", ".psb"]
+
+
+def has_unsaved_changes():
+ if _active_document():
+ return not lib.stub().is_saved()
+
+ return False
+
+
+def save_file(filepath):
+ _, ext = os.path.splitext(filepath)
+ lib.stub().saveAs(filepath, ext[1:], True)
+
+
+def open_file(filepath):
+ lib.stub().open(filepath)
+
+ return True
+
+
+def current_file():
+ try:
+ full_name = lib.stub().get_active_document_full_name()
+ if full_name and full_name != "null":
+ return os.path.normpath(full_name).replace("\\", "/")
+ except Exception:
+ pass
+
+ return None
+
+
+def work_root(session):
+ return os.path.normpath(session["AVALON_WORKDIR"]).replace("\\", "/")
diff --git a/openpype/hosts/photoshop/api/ws_stub.py b/openpype/hosts/photoshop/api/ws_stub.py
new file mode 100644
index 00000000000..2c4d0ad5fc8
--- /dev/null
+++ b/openpype/hosts/photoshop/api/ws_stub.py
@@ -0,0 +1,571 @@
+"""
+ Stub handling connection from server to client.
+ Used anywhere solution is calling client methods.
+"""
+import json
+import attr
+from wsrpc_aiohttp import WebSocketAsync
+
+from openpype.tools.adobe_webserver.app import WebServerTool
+
+
+@attr.s
+class PSItem(object):
+ """
+ Object denoting layer or group item in PS. Each item is created in
+ PS by any Loader, but contains same fields, which are being used
+ in later processing.
+ """
+ # metadata
+ id = attr.ib() # id created by AE, could be used for querying
+ name = attr.ib() # name of item
+ group = attr.ib(default=None) # item type (footage, folder, comp)
+ parents = attr.ib(factory=list)
+ visible = attr.ib(default=True)
+ type = attr.ib(default=None)
+ # all imported elements, single for
+ members = attr.ib(factory=list)
+ long_name = attr.ib(default=None)
+ color_code = attr.ib(default=None) # color code of layer
+ instance_id = attr.ib(default=None)
+
+ @property
+ def clean_name(self):
+ """Returns layer name without publish icon highlight
+
+ Returns:
+ (str)
+ """
+ return (self.name.replace(PhotoshopServerStub.PUBLISH_ICON, '')
+ .replace(PhotoshopServerStub.LOADED_ICON, ''))
+
+
+class PhotoshopServerStub:
+ """
+ Stub for calling function on client (Photoshop js) side.
+ Expects that client is already connected (started when avalon menu
+ is opened).
+ 'self.websocketserver.call' is used as async wrapper
+ """
+ PUBLISH_ICON = '\u2117 '
+ LOADED_ICON = '\u25bc'
+
+ def __init__(self):
+ self.websocketserver = WebServerTool.get_instance()
+ self.client = self.get_client()
+
+ @staticmethod
+ def get_client():
+ """
+ Return first connected client to WebSocket
+ TODO implement selection by Route
+ :return: client
+ """
+ clients = WebSocketAsync.get_clients()
+ client = None
+ if len(clients) > 0:
+ key = list(clients.keys())[0]
+ client = clients.get(key)
+
+ return client
+
+ def open(self, path):
+ """Open file located at 'path' (local).
+
+ Args:
+ path(string): file path locally
+ Returns: None
+ """
+ self.websocketserver.call(
+ self.client.call('Photoshop.open', path=path)
+ )
+
+ def read(self, layer, layers_meta=None):
+ """Parses layer metadata from Headline field of active document.
+
+ Args:
+ layer: (PSItem)
+ layers_meta: full list from Headline (for performance in loops)
+ Returns:
+ (dict) of layer metadata stored in PS file
+
+ Example:
+ {
+ 'id': 'pyblish.avalon.container',
+ 'loader': 'ImageLoader',
+ 'members': ['64'],
+ 'name': 'imageMainMiddle',
+ 'namespace': 'Hero_imageMainMiddle_001',
+ 'representation': '6203dc91e80934d9f6ee7d96',
+ 'schema': 'openpype:container-2.0'
+ }
+ """
+ if layers_meta is None:
+ layers_meta = self.get_layers_metadata()
+
+ for layer_meta in layers_meta:
+ layer_id = layer_meta.get("uuid") # legacy
+ if layer_meta.get("members"):
+ layer_id = layer_meta["members"][0]
+ if str(layer.id) == str(layer_id):
+ return layer_meta
+ print("Unable to find layer metadata for {}".format(layer.id))
+
+ def imprint(self, item_id, data, all_layers=None, items_meta=None):
+ """Save layer metadata to Headline field of active document
+
+ Stores metadata in format:
+ [{
+ "active":true,
+ "subset":"imageBG",
+ "family":"image",
+ "id":"pyblish.avalon.instance",
+ "asset":"Town",
+ "uuid": "8"
+ }] - for created instances
+ OR
+ [{
+ "schema": "openpype:container-2.0",
+ "id": "pyblish.avalon.instance",
+ "name": "imageMG",
+ "namespace": "Jungle_imageMG_001",
+ "loader": "ImageLoader",
+ "representation": "5fbfc0ee30a946093c6ff18a",
+ "members": [
+ "40"
+ ]
+ }] - for loaded instances
+
+ Args:
+ item_id (str):
+ data(string): json representation for single layer
+ all_layers (list of PSItem): for performance, could be
+ injected for usage in loop, if not, single call will be
+ triggered
+ items_meta(string): json representation from Headline
+ (for performance - provide only if imprint is in
+ loop - value should be same)
+ Returns: None
+ """
+ if not items_meta:
+ items_meta = self.get_layers_metadata()
+
+ # json.dumps writes integer values in a dictionary to string, so
+ # anticipating it here.
+ item_id = str(item_id)
+ is_new = True
+ result_meta = []
+ for item_meta in items_meta:
+ if ((item_meta.get('members') and
+ item_id == str(item_meta.get('members')[0])) or
+ item_meta.get("instance_id") == item_id):
+ is_new = False
+ if data:
+ item_meta.update(data)
+ result_meta.append(item_meta)
+ else:
+ result_meta.append(item_meta)
+
+ if is_new:
+ result_meta.append(data)
+
+ # Ensure only valid ids are stored.
+ if not all_layers:
+ all_layers = self.get_layers()
+ layer_ids = [layer.id for layer in all_layers]
+ cleaned_data = []
+
+ for item in result_meta:
+ if item.get("members"):
+ if int(item["members"][0]) not in layer_ids:
+ continue
+
+ cleaned_data.append(item)
+
+ payload = json.dumps(cleaned_data, indent=4)
+ self.websocketserver.call(
+ self.client.call('Photoshop.imprint', payload=payload)
+ )
+
+ def get_layers(self):
+ """Returns JSON document with all(?) layers in active document.
+
+ Returns:
+ Format of tuple: { 'id':'123',
+ 'name': 'My Layer 1',
+ 'type': 'GUIDE'|'FG'|'BG'|'OBJ'
+ 'visible': 'true'|'false'
+ """
+ res = self.websocketserver.call(
+ self.client.call('Photoshop.get_layers')
+ )
+
+ return self._to_records(res)
+
+ def get_layer(self, layer_id):
+ """
+ Returns PSItem for specific 'layer_id' or None if not found
+ Args:
+ layer_id (string): unique layer id, stored in 'uuid' field
+
+ Returns:
+ (PSItem) or None
+ """
+ layers = self.get_layers()
+ for layer in layers:
+ if str(layer.id) == str(layer_id):
+ return layer
+
+ def get_layers_in_layers(self, layers):
+ """Return all layers that belong to layers (might be groups).
+
+ Args:
+ layers :
+
+ Returns:
+
+ """
+ parent_ids = set([lay.id for lay in layers])
+
+ return self._get_layers_in_layers(parent_ids)
+
+ def get_layers_in_layers_ids(self, layers_ids, layers=None):
+ """Return all layers that belong to layers (might be groups).
+
+ Args:
+ layers_ids
+ layers :
+
+ Returns:
+
+ """
+ parent_ids = set(layers_ids)
+
+ return self._get_layers_in_layers(parent_ids, layers)
+
+ def _get_layers_in_layers(self, parent_ids, layers=None):
+ if not layers:
+ layers = self.get_layers()
+
+ all_layers = layers
+ ret = []
+
+ for layer in all_layers:
+ parents = set(layer.parents)
+ if len(parent_ids & parents) > 0:
+ ret.append(layer)
+ if layer.id in parent_ids:
+ ret.append(layer)
+
+ return ret
+
+ def create_group(self, name):
+ """Create new group (eg. LayerSet)
+
+ Returns:
+
+ """
+ enhanced_name = self.PUBLISH_ICON + name
+ ret = self.websocketserver.call(
+ self.client.call('Photoshop.create_group', name=enhanced_name)
+ )
+ # create group on PS is asynchronous, returns only id
+ return PSItem(id=ret, name=name, group=True)
+
+ def group_selected_layers(self, name):
+ """Group selected layers into new LayerSet (eg. group)
+
+ Returns:
+ (Layer)
+ """
+ enhanced_name = self.PUBLISH_ICON + name
+ res = self.websocketserver.call(
+ self.client.call(
+ 'Photoshop.group_selected_layers', name=enhanced_name
+ )
+ )
+ res = self._to_records(res)
+ if res:
+ rec = res.pop()
+ rec.name = rec.name.replace(self.PUBLISH_ICON, '')
+ return rec
+ raise ValueError("No group record returned")
+
+ def get_selected_layers(self):
+ """Get a list of actually selected layers.
+
+ Returns:
+ """
+ res = self.websocketserver.call(
+ self.client.call('Photoshop.get_selected_layers')
+ )
+ return self._to_records(res)
+
+ def select_layers(self, layers):
+ """Selects specified layers in Photoshop by its ids.
+
+ Args:
+ layers:
+ """
+ layers_id = [str(lay.id) for lay in layers]
+ self.websocketserver.call(
+ self.client.call(
+ 'Photoshop.select_layers',
+ layers=json.dumps(layers_id)
+ )
+ )
+
+ def get_active_document_full_name(self):
+ """Returns full name with path of active document via ws call
+
+ Returns(string):
+ full path with name
+ """
+ res = self.websocketserver.call(
+ self.client.call('Photoshop.get_active_document_full_name')
+ )
+
+ return res
+
+ def get_active_document_name(self):
+ """Returns just a name of active document via ws call
+
+ Returns(string):
+ file name
+ """
+ return self.websocketserver.call(
+ self.client.call('Photoshop.get_active_document_name')
+ )
+
+ def is_saved(self):
+ """Returns true if no changes in active document
+
+ Returns:
+
+ """
+ return self.websocketserver.call(
+ self.client.call('Photoshop.is_saved')
+ )
+
+ def save(self):
+ """Saves active document"""
+ self.websocketserver.call(
+ self.client.call('Photoshop.save')
+ )
+
+ def saveAs(self, image_path, ext, as_copy):
+ """Saves active document to psd (copy) or png or jpg
+
+ Args:
+ image_path(string): full local path
+ ext:
+ as_copy:
+ Returns: None
+ """
+ self.websocketserver.call(
+ self.client.call(
+ 'Photoshop.saveAs',
+ image_path=image_path,
+ ext=ext,
+ as_copy=as_copy
+ )
+ )
+
+ def set_visible(self, layer_id, visibility):
+ """Set layer with 'layer_id' to 'visibility'
+
+ Args:
+ layer_id:
+ visibility:
+ Returns: None
+ """
+ self.websocketserver.call(
+ self.client.call(
+ 'Photoshop.set_visible',
+ layer_id=layer_id,
+ visibility=visibility
+ )
+ )
+
+ def hide_all_others_layers(self, layers):
+ """hides all layers that are not part of the list or that are not
+ children of this list
+
+ Args:
+ layers (list): list of PSItem - highest hierarchy
+ """
+ extract_ids = set([ll.id for ll in self.get_layers_in_layers(layers)])
+
+ self.hide_all_others_layers_ids(extract_ids)
+
+ def hide_all_others_layers_ids(self, extract_ids, layers=None):
+ """hides all layers that are not part of the list or that are not
+ children of this list
+
+ Args:
+ extract_ids (list): list of integer that should be visible
+ layers (list) of PSItem (used for caching)
+ """
+ if not layers:
+ layers = self.get_layers()
+ for layer in layers:
+ if layer.visible and layer.id not in extract_ids:
+ self.set_visible(layer.id, False)
+
+ def get_layers_metadata(self):
+ """Reads layers metadata from Headline from active document in PS.
+ (Headline accessible by File > File Info)
+
+ Returns:
+ (list)
+ example:
+ {"8":{"active":true,"subset":"imageBG",
+ "family":"image","id":"pyblish.avalon.instance",
+ "asset":"Town"}}
+ 8 is layer(group) id - used for deletion, update etc.
+ """
+ res = self.websocketserver.call(self.client.call('Photoshop.read'))
+ layers_data = []
+ try:
+ if res:
+ layers_data = json.loads(res)
+ except json.decoder.JSONDecodeError:
+ raise ValueError("{} cannot be parsed, recreate meta".format(res))
+ # format of metadata changed from {} to [] because of standardization
+ # keep current implementation logic as its working
+ if isinstance(layers_data, dict):
+ for layer_id, layer_meta in layers_data.items():
+ if layer_meta.get("schema") != "openpype:container-2.0":
+ layer_meta["members"] = [str(layer_id)]
+ layers_data = list(layers_data.values())
+ return layers_data
+
+ def import_smart_object(self, path, layer_name, as_reference=False):
+ """Import the file at `path` as a smart object to active document.
+
+ Args:
+ path (str): File path to import.
+ layer_name (str): Unique layer name to differentiate how many times
+ same smart object was loaded
+ as_reference (bool): pull in content or reference
+ """
+ enhanced_name = self.LOADED_ICON + layer_name
+ res = self.websocketserver.call(
+ self.client.call(
+ 'Photoshop.import_smart_object',
+ path=path,
+ name=enhanced_name,
+ as_reference=as_reference
+ )
+ )
+ rec = self._to_records(res).pop()
+ if rec:
+ rec.name = rec.name.replace(self.LOADED_ICON, '')
+ return rec
+
+ def replace_smart_object(self, layer, path, layer_name):
+ """Replace the smart object `layer` with file at `path`
+
+ Args:
+ layer (PSItem):
+ path (str): File to import.
+ layer_name (str): Unique layer name to differentiate how many times
+ same smart object was loaded
+ """
+ enhanced_name = self.LOADED_ICON + layer_name
+ self.websocketserver.call(
+ self.client.call(
+ 'Photoshop.replace_smart_object',
+ layer_id=layer.id,
+ path=path,
+ name=enhanced_name
+ )
+ )
+
+ def delete_layer(self, layer_id):
+ """Deletes specific layer by it's id.
+
+ Args:
+ layer_id (int): id of layer to delete
+ """
+ self.websocketserver.call(
+ self.client.call('Photoshop.delete_layer', layer_id=layer_id)
+ )
+
+ def rename_layer(self, layer_id, name):
+ """Renames specific layer by it's id.
+
+ Args:
+ layer_id (int): id of layer to delete
+ name (str): new name
+ """
+ self.websocketserver.call(
+ self.client.call(
+ 'Photoshop.rename_layer',
+ layer_id=layer_id,
+ name=name
+ )
+ )
+
+ def remove_instance(self, instance_id):
+ cleaned_data = []
+
+ for item in self.get_layers_metadata():
+ inst_id = item.get("instance_id") or item.get("uuid")
+ if inst_id != instance_id:
+ cleaned_data.append(item)
+
+ payload = json.dumps(cleaned_data, indent=4)
+
+ self.websocketserver.call(
+ self.client.call('Photoshop.imprint', payload=payload)
+ )
+
+ def get_extension_version(self):
+ """Returns version number of installed extension."""
+ return self.websocketserver.call(
+ self.client.call('Photoshop.get_extension_version')
+ )
+
+ def close(self):
+ """Shutting down PS and process too.
+
+ For webpublishing only.
+ """
+ # TODO change client.call to method with checks for client
+ self.websocketserver.call(self.client.call('Photoshop.close'))
+
+ def _to_records(self, res):
+ """Converts string json representation into list of PSItem for
+ dot notation access to work.
+
+ Args:
+ res (string): valid json
+
+ Returns:
+
+ """
+ try:
+ layers_data = json.loads(res)
+ except json.decoder.JSONDecodeError:
+ raise ValueError("Received broken JSON {}".format(res))
+ ret = []
+
+ # convert to AEItem to use dot donation
+ if isinstance(layers_data, dict):
+ layers_data = [layers_data]
+ for d in layers_data:
+ # currently implemented and expected fields
+ ret.append(PSItem(
+ d.get('id'),
+ d.get('name'),
+ d.get('group'),
+ d.get('parents'),
+ d.get('visible'),
+ d.get('type'),
+ d.get('members'),
+ d.get('long_name'),
+ d.get("color_code"),
+ d.get("instance_id")
+ ))
+ return ret
diff --git a/openpype/hosts/photoshop/plugins/create/create_image.py b/openpype/hosts/photoshop/plugins/create/create_image.py
index 657d41aa93f..2cfbfa8778c 100644
--- a/openpype/hosts/photoshop/plugins/create/create_image.py
+++ b/openpype/hosts/photoshop/plugins/create/create_image.py
@@ -1,103 +1,168 @@
-from Qt import QtWidgets
-import openpype.api
-from avalon import photoshop
+import re
+from openpype.hosts.photoshop import api
+from openpype.lib import BoolDef
+from openpype.pipeline import (
+ Creator,
+ CreatedInstance,
+ legacy_io
+)
+from openpype.lib import prepare_template_data
+from openpype.pipeline.create import SUBSET_NAME_ALLOWED_SYMBOLS
-class CreateImage(openpype.api.Creator):
- """Image folder for publish."""
- name = "imageDefault"
+class ImageCreator(Creator):
+ """Creates image instance for publishing."""
+ identifier = "image"
label = "Image"
family = "image"
- defaults = ["Main"]
-
- def process(self):
- groups = []
- layers = []
- create_group = False
-
- stub = photoshop.stub()
- useSelection = False
- if (self.options or {}).get("useSelection"):
- useSelection = True
- multiple_instances = False
- selection = stub.get_selected_layers()
- self.log.info("selection {}".format(selection))
- if len(selection) > 1:
- # Ask user whether to create one image or image per selected
- # item.
- msg_box = QtWidgets.QMessageBox()
- msg_box.setIcon(QtWidgets.QMessageBox.Warning)
- msg_box.setText(
- "Multiple layers selected."
- "\nDo you want to make one image per layer?"
- )
- msg_box.setStandardButtons(
- QtWidgets.QMessageBox.Yes |
- QtWidgets.QMessageBox.No |
- QtWidgets.QMessageBox.Cancel
+ description = "Image creator"
+
+ def collect_instances(self):
+ for instance_data in api.list_instances():
+ # legacy instances have family=='image'
+ creator_id = (instance_data.get("creator_identifier") or
+ instance_data.get("family"))
+
+ if creator_id == self.identifier:
+ instance_data = self._handle_legacy(instance_data)
+ layer = api.stub().get_layer(instance_data["members"][0])
+ instance_data["layer"] = layer
+ instance = CreatedInstance.from_existing(
+ instance_data, self
)
- ret = msg_box.exec_()
- if ret == QtWidgets.QMessageBox.Yes:
- multiple_instances = True
- elif ret == QtWidgets.QMessageBox.Cancel:
- return
-
- if multiple_instances:
- for item in selection:
- if item.group:
- groups.append(item)
- else:
- layers.append(item)
- else:
- group = stub.group_selected_layers(self.name)
- groups.append(group)
-
- elif len(selection) == 1:
- # One selected item. Use group if its a LayerSet (group), else
- # create a new group.
- if selection[0].group:
- groups.append(selection[0])
- else:
- layers.append(selection[0])
- elif len(selection) == 0:
- # No selection creates an empty group.
- create_group = True
+ self._add_instance_to_context(instance)
+
+ def create(self, subset_name_from_ui, data, pre_create_data):
+ groups_to_create = []
+ top_layers_to_wrap = []
+ create_empty_group = False
+
+ stub = api.stub() # only after PS is up
+ top_level_selected_items = stub.get_selected_layers()
+ if pre_create_data.get("use_selection"):
+ only_single_item_selected = len(top_level_selected_items) == 1
+ if (
+ only_single_item_selected or
+ pre_create_data.get("create_multiple")):
+ for selected_item in top_level_selected_items:
+ if selected_item.group:
+ groups_to_create.append(selected_item)
+ else:
+ top_layers_to_wrap.append(selected_item)
+ else:
+ group = stub.group_selected_layers(subset_name_from_ui)
+ groups_to_create.append(group)
else:
stub.select_layers(stub.get_layers())
- group = stub.group_selected_layers(self.name)
- groups.append(group)
+ try:
+ group = stub.group_selected_layers(subset_name_from_ui)
+ except:
+ raise ValueError("Cannot group locked Bakcground layer!")
+ groups_to_create.append(group)
- if create_group:
- group = stub.create_group(self.name)
- groups.append(group)
+ if not groups_to_create and not top_layers_to_wrap:
+ group = stub.create_group(subset_name_from_ui)
+ groups_to_create.append(group)
- for layer in layers:
+ # wrap each top level layer into separate new group
+ for layer in top_layers_to_wrap:
stub.select_layers([layer])
group = stub.group_selected_layers(layer.name)
- groups.append(group)
+ groups_to_create.append(group)
- for group in groups:
- long_names = []
- group.name = group.name.replace(stub.PUBLISH_ICON, ''). \
- replace(stub.LOADED_ICON, '')
+ layer_name = ''
+ creating_multiple_groups = len(groups_to_create) > 1
+ for group in groups_to_create:
+ subset_name = subset_name_from_ui # reset to name from creator UI
+ layer_names_in_hierarchy = []
+ created_group_name = self._clean_highlights(stub, group.name)
- if useSelection:
- subset_name = self.data["subset"] + group.name
- else:
- # use value provided by user from Creator
- subset_name = self.data["subset"]
+ if creating_multiple_groups:
+ layer_name = re.sub(
+ "[^{}]+".format(SUBSET_NAME_ALLOWED_SYMBOLS),
+ "",
+ group.name
+ )
+ if "{layer}" not in subset_name.lower():
+ subset_name += "{Layer}"
+
+ layer_fill = prepare_template_data({"layer": layer_name})
+ subset_name = subset_name.format(**layer_fill)
if group.long_name:
for directory in group.long_name[::-1]:
- name = directory.replace(stub.PUBLISH_ICON, '').\
- replace(stub.LOADED_ICON, '')
- long_names.append(name)
-
- self.data.update({"subset": subset_name})
- self.data.update({"uuid": str(group.id)})
- self.data.update({"long_name": "_".join(long_names)})
- stub.imprint(group, self.data)
+ name = self._clean_highlights(stub, directory)
+ layer_names_in_hierarchy.append(name)
+
+ data.update({"subset": subset_name})
+ data.update({"members": [str(group.id)]})
+ data.update({"long_name": "_".join(layer_names_in_hierarchy)})
+
+ new_instance = CreatedInstance(self.family, subset_name, data,
+ self)
+
+ stub.imprint(new_instance.get("instance_id"),
+ new_instance.data_to_store())
+ self._add_instance_to_context(new_instance)
# reusing existing group, need to rename afterwards
- if not create_group:
- stub.rename_layer(group.id, stub.PUBLISH_ICON + group.name)
+ if not create_empty_group:
+ stub.rename_layer(group.id,
+ stub.PUBLISH_ICON + created_group_name)
+
+ def update_instances(self, update_list):
+ self.log.debug("update_list:: {}".format(update_list))
+ for created_inst, _changes in update_list:
+ if created_inst.get("layer"):
+ # not storing PSItem layer to metadata
+ created_inst.pop("layer")
+ api.stub().imprint(created_inst.get("instance_id"),
+ created_inst.data_to_store())
+
+ def remove_instances(self, instances):
+ for instance in instances:
+ api.remove_instance(instance)
+ self._remove_instance_from_context(instance)
+
+ def get_default_variants(self):
+ return [
+ "Main"
+ ]
+
+ def get_pre_create_attr_defs(self):
+ output = [
+ BoolDef("use_selection", default=True,
+ label="Create only for selected"),
+ BoolDef("create_multiple",
+ default=True,
+ label="Create separate instance for each selected")
+ ]
+ return output
+
+ def get_detail_description(self):
+ return """Creator for Image instances"""
+
+ def _handle_legacy(self, instance_data):
+ """Converts old instances to new format."""
+ if not instance_data.get("members"):
+ instance_data["members"] = [instance_data.get("uuid")]
+
+ if instance_data.get("uuid"):
+ # uuid not needed, replaced with unique instance_id
+ api.stub().remove_instance(instance_data.get("uuid"))
+ instance_data.pop("uuid")
+
+ if not instance_data.get("task"):
+ instance_data["task"] = legacy_io.Session.get("AVALON_TASK")
+
+ if not instance_data.get("variant"):
+ instance_data["variant"] = ''
+
+ return instance_data
+
+ def _clean_highlights(self, stub, item):
+ return item.replace(stub.PUBLISH_ICON, '').replace(stub.LOADED_ICON,
+ '')
+ @classmethod
+ def get_dynamic_data(cls, *args, **kwargs):
+ return {"layer": "{layer}"}
diff --git a/openpype/hosts/photoshop/plugins/create/create_legacy_image.py b/openpype/hosts/photoshop/plugins/create/create_legacy_image.py
new file mode 100644
index 00000000000..2792a775e06
--- /dev/null
+++ b/openpype/hosts/photoshop/plugins/create/create_legacy_image.py
@@ -0,0 +1,119 @@
+import re
+
+from Qt import QtWidgets
+from openpype.pipeline import create
+from openpype.hosts.photoshop import api as photoshop
+
+from openpype.lib import prepare_template_data
+from openpype.pipeline.create import SUBSET_NAME_ALLOWED_SYMBOLS
+
+
+class CreateImage(create.LegacyCreator):
+ """Image folder for publish."""
+
+ name = "imageDefault"
+ label = "Image"
+ family = "image"
+ defaults = ["Main"]
+
+ def process(self):
+ groups = []
+ layers = []
+ create_group = False
+
+ stub = photoshop.stub()
+ if (self.options or {}).get("useSelection"):
+ multiple_instances = False
+ selection = stub.get_selected_layers()
+ self.log.info("selection {}".format(selection))
+ if len(selection) > 1:
+ # Ask user whether to create one image or image per selected
+ # item.
+ msg_box = QtWidgets.QMessageBox()
+ msg_box.setIcon(QtWidgets.QMessageBox.Warning)
+ msg_box.setText(
+ "Multiple layers selected."
+ "\nDo you want to make one image per layer?"
+ )
+ msg_box.setStandardButtons(
+ QtWidgets.QMessageBox.Yes |
+ QtWidgets.QMessageBox.No |
+ QtWidgets.QMessageBox.Cancel
+ )
+ ret = msg_box.exec_()
+ if ret == QtWidgets.QMessageBox.Yes:
+ multiple_instances = True
+ elif ret == QtWidgets.QMessageBox.Cancel:
+ return
+
+ if multiple_instances:
+ for item in selection:
+ if item.group:
+ groups.append(item)
+ else:
+ layers.append(item)
+ else:
+ group = stub.group_selected_layers(self.name)
+ groups.append(group)
+
+ elif len(selection) == 1:
+ # One selected item. Use group if its a LayerSet (group), else
+ # create a new group.
+ if selection[0].group:
+ groups.append(selection[0])
+ else:
+ layers.append(selection[0])
+ elif len(selection) == 0:
+ # No selection creates an empty group.
+ create_group = True
+ else:
+ group = stub.create_group(self.name)
+ groups.append(group)
+
+ if create_group:
+ group = stub.create_group(self.name)
+ groups.append(group)
+
+ for layer in layers:
+ stub.select_layers([layer])
+ group = stub.group_selected_layers(layer.name)
+ groups.append(group)
+
+ creator_subset_name = self.data["subset"]
+ layer_name = ''
+ for group in groups:
+ long_names = []
+ group.name = group.name.replace(stub.PUBLISH_ICON, ''). \
+ replace(stub.LOADED_ICON, '')
+
+ subset_name = creator_subset_name
+ if len(groups) > 1:
+ layer_name = re.sub(
+ "[^{}]+".format(SUBSET_NAME_ALLOWED_SYMBOLS),
+ "",
+ group.name
+ )
+ if "{layer}" not in subset_name.lower():
+ subset_name += "{Layer}"
+
+ layer_fill = prepare_template_data({"layer": layer_name})
+ subset_name = subset_name.format(**layer_fill)
+
+ if group.long_name:
+ for directory in group.long_name[::-1]:
+ name = directory.replace(stub.PUBLISH_ICON, '').\
+ replace(stub.LOADED_ICON, '')
+ long_names.append(name)
+
+ self.data.update({"subset": subset_name})
+ self.data.update({"uuid": str(group.id)})
+ self.data.update({"members": [str(group.id)]})
+ self.data.update({"long_name": "_".join(long_names)})
+ stub.imprint(group, self.data)
+ # reusing existing group, need to rename afterwards
+ if not create_group:
+ stub.rename_layer(group.id, stub.PUBLISH_ICON + group.name)
+
+ @classmethod
+ def get_dynamic_data(cls, *args, **kwargs):
+ return {"layer": "{layer}"}
diff --git a/openpype/hosts/photoshop/plugins/create/workfile_creator.py b/openpype/hosts/photoshop/plugins/create/workfile_creator.py
new file mode 100644
index 00000000000..e79d16d154f
--- /dev/null
+++ b/openpype/hosts/photoshop/plugins/create/workfile_creator.py
@@ -0,0 +1,78 @@
+import openpype.hosts.photoshop.api as api
+from openpype.client import get_asset_by_name
+from openpype.pipeline import (
+ AutoCreator,
+ CreatedInstance,
+ legacy_io
+)
+
+
+class PSWorkfileCreator(AutoCreator):
+ identifier = "workfile"
+ family = "workfile"
+
+ default_variant = "Main"
+
+ def get_instance_attr_defs(self):
+ return []
+
+ def collect_instances(self):
+ for instance_data in api.list_instances():
+ creator_id = instance_data.get("creator_identifier")
+ if creator_id == self.identifier:
+ subset_name = instance_data["subset"]
+ instance = CreatedInstance(
+ self.family, subset_name, instance_data, self
+ )
+ self._add_instance_to_context(instance)
+
+ def update_instances(self, update_list):
+ # nothing to change on workfiles
+ pass
+
+ def create(self, options=None):
+ existing_instance = None
+ for instance in self.create_context.instances:
+ if instance.family == self.family:
+ existing_instance = instance
+ break
+
+ project_name = legacy_io.Session["AVALON_PROJECT"]
+ asset_name = legacy_io.Session["AVALON_ASSET"]
+ task_name = legacy_io.Session["AVALON_TASK"]
+ host_name = legacy_io.Session["AVALON_APP"]
+ if existing_instance is None:
+ asset_doc = get_asset_by_name(project_name, asset_name)
+ subset_name = self.get_subset_name(
+ self.default_variant, task_name, asset_doc,
+ project_name, host_name
+ )
+ data = {
+ "asset": asset_name,
+ "task": task_name,
+ "variant": self.default_variant
+ }
+ data.update(self.get_dynamic_data(
+ self.default_variant, task_name, asset_doc,
+ project_name, host_name
+ ))
+
+ new_instance = CreatedInstance(
+ self.family, subset_name, data, self
+ )
+ self._add_instance_to_context(new_instance)
+ api.stub().imprint(new_instance.get("instance_id"),
+ new_instance.data_to_store())
+
+ elif (
+ existing_instance["asset"] != asset_name
+ or existing_instance["task"] != task_name
+ ):
+ asset_doc = get_asset_by_name(project_name, asset_name)
+ subset_name = self.get_subset_name(
+ self.default_variant, task_name, asset_doc,
+ project_name, host_name
+ )
+ existing_instance["asset"] = asset_name
+ existing_instance["task"] = task_name
+ existing_instance["subset"] = subset_name
diff --git a/openpype/hosts/photoshop/plugins/lib.py b/openpype/hosts/photoshop/plugins/lib.py
deleted file mode 100644
index 74aff061147..00000000000
--- a/openpype/hosts/photoshop/plugins/lib.py
+++ /dev/null
@@ -1,26 +0,0 @@
-import re
-
-
-def get_unique_layer_name(layers, asset_name, subset_name):
- """
- Gets all layer names and if 'asset_name_subset_name' is present, it
- increases suffix by 1 (eg. creates unique layer name - for Loader)
- Args:
- layers (list) of dict with layers info (name, id etc.)
- asset_name (string):
- subset_name (string):
-
- Returns:
- (string): name_00X (without version)
- """
- name = "{}_{}".format(asset_name, subset_name)
- names = {}
- for layer in layers:
- layer_name = re.sub(r'_\d{3}$', '', layer.name)
- if layer_name in names.keys():
- names[layer_name] = names[layer_name] + 1
- else:
- names[layer_name] = 1
- occurrences = names.get(name, 0)
-
- return "{}_{:0>3d}".format(name, occurrences + 1)
diff --git a/openpype/hosts/photoshop/plugins/load/load_image.py b/openpype/hosts/photoshop/plugins/load/load_image.py
index 981a1ed2049..91a97877811 100644
--- a/openpype/hosts/photoshop/plugins/load/load_image.py
+++ b/openpype/hosts/photoshop/plugins/load/load_image.py
@@ -1,12 +1,11 @@
import re
-from avalon import api, photoshop
+from openpype.pipeline import get_representation_path
+from openpype.hosts.photoshop import api as photoshop
+from openpype.hosts.photoshop.api import get_unique_layer_name
-from openpype.hosts.photoshop.plugins.lib import get_unique_layer_name
-stub = photoshop.stub()
-
-class ImageLoader(api.Loader):
+class ImageLoader(photoshop.PhotoshopLoader):
"""Load images
Stores the imported asset in a container named after the asset.
@@ -16,11 +15,14 @@ class ImageLoader(api.Loader):
representations = ["*"]
def load(self, context, name=None, namespace=None, data=None):
- layer_name = get_unique_layer_name(stub.get_layers(),
- context["asset"]["name"],
- name)
+ stub = self.get_stub()
+ layer_name = get_unique_layer_name(
+ stub.get_layers(),
+ context["asset"]["name"],
+ name
+ )
with photoshop.maintained_selection():
- layer = self.import_layer(self.fname, layer_name)
+ layer = self.import_layer(self.fname, layer_name, stub)
self[:] = [layer]
namespace = namespace or layer_name
@@ -35,6 +37,8 @@ def load(self, context, name=None, namespace=None, data=None):
def update(self, container, representation):
""" Switch asset or change version """
+ stub = self.get_stub()
+
layer = container.pop("layer")
context = representation.get("context", {})
@@ -44,20 +48,20 @@ def update(self, container, representation):
layer_name = "{}_{}".format(context["asset"], context["subset"])
# switching assets
if namespace_from_container != layer_name:
- layer_name = get_unique_layer_name(stub.get_layers(),
- context["asset"],
- context["subset"])
+ layer_name = get_unique_layer_name(
+ stub.get_layers(), context["asset"], context["subset"]
+ )
else: # switching version - keep same name
layer_name = container["namespace"]
- path = api.get_representation_path(representation)
+ path = get_representation_path(representation)
with photoshop.maintained_selection():
stub.replace_smart_object(
layer, path, layer_name
)
stub.imprint(
- layer, {"representation": str(representation["_id"])}
+ layer.id, {"representation": str(representation["_id"])}
)
def remove(self, container):
@@ -66,12 +70,14 @@ def remove(self, container):
Args:
container (dict): container to be removed - used to get layer_id
"""
+ stub = self.get_stub()
+
layer = container.pop("layer")
- stub.imprint(layer, {})
+ stub.imprint(layer.id, {})
stub.delete_layer(layer.id)
def switch(self, container, representation):
self.update(container, representation)
- def import_layer(self, file_name, layer_name):
+ def import_layer(self, file_name, layer_name, stub):
return stub.import_smart_object(file_name, layer_name)
diff --git a/openpype/hosts/photoshop/plugins/load/load_image_from_sequence.py b/openpype/hosts/photoshop/plugins/load/load_image_from_sequence.py
index 8704627b123..c25c5a8f2cf 100644
--- a/openpype/hosts/photoshop/plugins/load/load_image_from_sequence.py
+++ b/openpype/hosts/photoshop/plugins/load/load_image_from_sequence.py
@@ -1,18 +1,13 @@
import os
-from avalon import api
-from avalon import photoshop
-from avalon.pipeline import get_representation_path_from_context
-from avalon.vendor import qargparse
+import qargparse
-from openpype.lib import Anatomy
-from openpype.hosts.photoshop.plugins.lib import get_unique_layer_name
+from openpype.hosts.photoshop import api as photoshop
+from openpype.hosts.photoshop.api import get_unique_layer_name
-stub = photoshop.stub()
-
-class ImageFromSequenceLoader(api.Loader):
- """ Load specifing image from sequence
+class ImageFromSequenceLoader(photoshop.PhotoshopLoader):
+ """ Load specific image from sequence
Used only as quick load of reference file from a sequence.
@@ -35,15 +30,16 @@ class ImageFromSequenceLoader(api.Loader):
def load(self, context, name=None, namespace=None, data=None):
if data.get("frame"):
- self.fname = os.path.join(os.path.dirname(self.fname),
- data["frame"])
+ self.fname = os.path.join(
+ os.path.dirname(self.fname), data["frame"]
+ )
if not os.path.exists(self.fname):
return
- stub = photoshop.stub()
- layer_name = get_unique_layer_name(stub.get_layers(),
- context["asset"]["name"],
- name)
+ stub = self.get_stub()
+ layer_name = get_unique_layer_name(
+ stub.get_layers(), context["asset"]["name"], name
+ )
with photoshop.maintained_selection():
layer = stub.import_smart_object(self.fname, layer_name)
@@ -66,7 +62,7 @@ def get_options(cls, repre_contexts):
"""
files = []
for context in repre_contexts:
- fname = get_representation_path_from_context(context)
+ fname = cls.filepath_from_context(context)
_, file_extension = os.path.splitext(fname)
for file_name in os.listdir(os.path.dirname(fname)):
@@ -95,4 +91,3 @@ def update(self, container, representation):
def remove(self, container):
"""No update possible, not containerized."""
pass
-
diff --git a/openpype/hosts/photoshop/plugins/load/load_reference.py b/openpype/hosts/photoshop/plugins/load/load_reference.py
index 0cb4e4a69f3..1f32a5d23c9 100644
--- a/openpype/hosts/photoshop/plugins/load/load_reference.py
+++ b/openpype/hosts/photoshop/plugins/load/load_reference.py
@@ -1,30 +1,29 @@
import re
-from avalon import api, photoshop
+from openpype.pipeline import get_representation_path
+from openpype.hosts.photoshop import api as photoshop
+from openpype.hosts.photoshop.api import get_unique_layer_name
-from openpype.hosts.photoshop.plugins.lib import get_unique_layer_name
-stub = photoshop.stub()
-
-
-class ReferenceLoader(api.Loader):
+class ReferenceLoader(photoshop.PhotoshopLoader):
"""Load reference images
- Stores the imported asset in a container named after the asset.
+ Stores the imported asset in a container named after the asset.
- Inheriting from 'load_image' didn't work because of
- "Cannot write to closing transport", possible refactor.
+ Inheriting from 'load_image' didn't work because of
+ "Cannot write to closing transport", possible refactor.
"""
families = ["image", "render"]
representations = ["*"]
def load(self, context, name=None, namespace=None, data=None):
- layer_name = get_unique_layer_name(stub.get_layers(),
- context["asset"]["name"],
- name)
+ stub = self.get_stub()
+ layer_name = get_unique_layer_name(
+ stub.get_layers(), context["asset"]["name"], name
+ )
with photoshop.maintained_selection():
- layer = self.import_layer(self.fname, layer_name)
+ layer = self.import_layer(self.fname, layer_name, stub)
self[:] = [layer]
namespace = namespace or layer_name
@@ -39,6 +38,7 @@ def load(self, context, name=None, namespace=None, data=None):
def update(self, container, representation):
""" Switch asset or change version """
+ stub = self.get_stub()
layer = container.pop("layer")
context = representation.get("context", {})
@@ -48,35 +48,37 @@ def update(self, container, representation):
layer_name = "{}_{}".format(context["asset"], context["subset"])
# switching assets
if namespace_from_container != layer_name:
- layer_name = get_unique_layer_name(stub.get_layers(),
- context["asset"],
- context["subset"])
+ layer_name = get_unique_layer_name(
+ stub.get_layers(), context["asset"], context["subset"]
+ )
else: # switching version - keep same name
layer_name = container["namespace"]
- path = api.get_representation_path(representation)
+ path = get_representation_path(representation)
with photoshop.maintained_selection():
stub.replace_smart_object(
layer, path, layer_name
)
stub.imprint(
- layer, {"representation": str(representation["_id"])}
+ layer.id, {"representation": str(representation["_id"])}
)
def remove(self, container):
- """
- Removes element from scene: deletes layer + removes from Headline
+ """Removes element from scene: deletes layer + removes from Headline
+
Args:
container (dict): container to be removed - used to get layer_id
"""
+ stub = self.get_stub()
layer = container.pop("layer")
- stub.imprint(layer, {})
+ stub.imprint(layer.id, {})
stub.delete_layer(layer.id)
def switch(self, container, representation):
self.update(container, representation)
- def import_layer(self, file_name, layer_name):
- return stub.import_smart_object(file_name, layer_name,
- as_reference=True)
+ def import_layer(self, file_name, layer_name, stub):
+ return stub.import_smart_object(
+ file_name, layer_name, as_reference=True
+ )
diff --git a/openpype/hosts/photoshop/plugins/publish/closePS.py b/openpype/hosts/photoshop/plugins/publish/closePS.py
index 2f0eab0ee5b..b4ded96001c 100644
--- a/openpype/hosts/photoshop/plugins/publish/closePS.py
+++ b/openpype/hosts/photoshop/plugins/publish/closePS.py
@@ -4,7 +4,7 @@
import pyblish.api
-from avalon import photoshop
+from openpype.hosts.photoshop import api as photoshop
class ClosePS(pyblish.api.ContextPlugin):
diff --git a/openpype/hosts/photoshop/plugins/publish/collect_batch_data.py b/openpype/hosts/photoshop/plugins/publish/collect_batch_data.py
new file mode 100644
index 00000000000..5d50a789148
--- /dev/null
+++ b/openpype/hosts/photoshop/plugins/publish/collect_batch_data.py
@@ -0,0 +1,77 @@
+"""Parses batch context from json and continues in publish process.
+
+Provides:
+ context -> Loaded batch file.
+ - asset
+ - task (task name)
+ - taskType
+ - project_name
+ - variant
+
+Code is practically copy of `openype/hosts/webpublish/collect_batch_data` as
+webpublisher should be eventually ejected as an addon, eg. mentioned plugin
+shouldn't be pushed into general publish plugins.
+"""
+
+import os
+
+import pyblish.api
+
+from openpype.pipeline import legacy_io
+from openpype_modules.webpublisher.lib import (
+ get_batch_asset_task_info,
+ parse_json
+)
+
+
+class CollectBatchData(pyblish.api.ContextPlugin):
+ """Collect batch data from json stored in 'OPENPYPE_PUBLISH_DATA' env dir.
+
+ The directory must contain 'manifest.json' file where batch data should be
+ stored.
+ """
+ # must be really early, context values are only in json file
+ order = pyblish.api.CollectorOrder - 0.495
+ label = "Collect batch data"
+ hosts = ["photoshop"]
+ targets = ["remotepublish"]
+
+ def process(self, context):
+ self.log.info("CollectBatchData")
+ batch_dir = os.environ.get("OPENPYPE_PUBLISH_DATA")
+ if os.environ.get("IS_TEST"):
+ self.log.debug("Automatic testing, no batch data, skipping")
+ return
+
+ assert batch_dir, (
+ "Missing `OPENPYPE_PUBLISH_DATA`")
+
+ assert os.path.exists(batch_dir), \
+ "Folder {} doesn't exist".format(batch_dir)
+
+ project_name = os.environ.get("AVALON_PROJECT")
+ if project_name is None:
+ raise AssertionError(
+ "Environment `AVALON_PROJECT` was not found."
+ "Could not set project `root` which may cause issues."
+ )
+
+ batch_data = parse_json(os.path.join(batch_dir, "manifest.json"))
+
+ context.data["batchDir"] = batch_dir
+ context.data["batchData"] = batch_data
+
+ asset_name, task_name, task_type = get_batch_asset_task_info(
+ batch_data["context"]
+ )
+
+ os.environ["AVALON_ASSET"] = asset_name
+ os.environ["AVALON_TASK"] = task_name
+ legacy_io.Session["AVALON_ASSET"] = asset_name
+ legacy_io.Session["AVALON_TASK"] = task_name
+
+ context.data["asset"] = asset_name
+ context.data["task"] = task_name
+ context.data["taskType"] = task_type
+ context.data["project_name"] = project_name
+ context.data["variant"] = batch_data["variant"]
diff --git a/openpype/hosts/photoshop/plugins/publish/collect_color_coded_instances.py b/openpype/hosts/photoshop/plugins/publish/collect_color_coded_instances.py
new file mode 100644
index 00000000000..c157c932fd2
--- /dev/null
+++ b/openpype/hosts/photoshop/plugins/publish/collect_color_coded_instances.py
@@ -0,0 +1,231 @@
+import os
+import re
+
+import pyblish.api
+
+from openpype.lib import prepare_template_data
+from openpype.hosts.photoshop import api as photoshop
+from openpype.settings import get_project_settings
+
+
+class CollectColorCodedInstances(pyblish.api.ContextPlugin):
+ """Creates instances for layers marked by configurable color.
+
+ Used in remote publishing when artists marks publishable layers by color-
+ coding. Top level layers (group) must be marked by specific color to be
+ published as an instance of 'image' family.
+
+ Can add group for all publishable layers to allow creation of flattened
+ image. (Cannot contain special background layer as it cannot be grouped!)
+
+ Based on value `create_flatten_image` from Settings:
+ - "yes": create flattened 'image' subset of all publishable layers + create
+ 'image' subset per publishable layer
+ - "only": create ONLY flattened 'image' subset of all publishable layers
+ - "no": do not create flattened 'image' subset at all,
+ only separate subsets per marked layer.
+
+ Identifier:
+ id (str): "pyblish.avalon.instance"
+ """
+ order = pyblish.api.CollectorOrder + 0.100
+
+ label = "Instances"
+ order = pyblish.api.CollectorOrder
+ hosts = ["photoshop"]
+ targets = ["remotepublish"]
+
+ # configurable by Settings
+ color_code_mapping = []
+ # TODO check if could be set globally, probably doesn't make sense when
+ # flattened template cannot
+ subset_template_name = ""
+ create_flatten_image = "no"
+ flatten_subset_template = ""
+
+ def process(self, context):
+ self.log.info("CollectColorCodedInstances")
+ batch_dir = os.environ.get("OPENPYPE_PUBLISH_DATA")
+ if (os.environ.get("IS_TEST") and
+ (not batch_dir or not os.path.exists(batch_dir))):
+ self.log.debug("Automatic testing, no batch data, skipping")
+ return
+
+ existing_subset_names = self._get_existing_subset_names(context)
+
+ # from CollectBatchData
+ asset_name = context.data["asset"]
+ task_name = context.data["task"]
+ variant = context.data["variant"]
+ project_name = context.data["projectEntity"]["name"]
+
+ naming_conventions = get_project_settings(project_name).get(
+ "photoshop", {}).get(
+ "publish", {}).get(
+ "ValidateNaming", {})
+
+ stub = photoshop.stub()
+ layers = stub.get_layers()
+
+ publishable_layers = []
+ created_instances = []
+ family_from_settings = None
+ for layer in layers:
+ self.log.debug("Layer:: {}".format(layer))
+ if layer.parents:
+ self.log.debug("!!! Not a top layer, skip")
+ continue
+
+ if not layer.visible:
+ self.log.debug("Not visible, skip")
+ continue
+
+ resolved_family, resolved_subset_template = self._resolve_mapping(
+ layer
+ )
+
+ if not resolved_subset_template or not resolved_family:
+ self.log.debug("!!! Not found family or template, skip")
+ continue
+
+ if not family_from_settings:
+ family_from_settings = resolved_family
+
+ fill_pairs = {
+ "variant": variant,
+ "family": resolved_family,
+ "task": task_name,
+ "layer": layer.clean_name
+ }
+
+ subset = resolved_subset_template.format(
+ **prepare_template_data(fill_pairs))
+
+ subset = self._clean_subset_name(stub, naming_conventions,
+ subset, layer)
+
+ if subset in existing_subset_names:
+ self.log.info(
+ "Subset {} already created, skipping.".format(subset))
+ continue
+
+ if self.create_flatten_image != "flatten_only":
+ instance = self._create_instance(context, layer,
+ resolved_family,
+ asset_name, subset, task_name)
+ created_instances.append(instance)
+
+ existing_subset_names.append(subset)
+ publishable_layers.append(layer)
+
+ if self.create_flatten_image != "no" and publishable_layers:
+ self.log.debug("create_flatten_image")
+ if not self.flatten_subset_template:
+ self.log.warning("No template for flatten image")
+ return
+
+ fill_pairs.pop("layer")
+ subset = self.flatten_subset_template.format(
+ **prepare_template_data(fill_pairs))
+
+ first_layer = publishable_layers[0] # dummy layer
+ first_layer.name = subset
+ family = family_from_settings # inherit family
+ instance = self._create_instance(context, first_layer,
+ family,
+ asset_name, subset, task_name)
+ instance.data["ids"] = [layer.id for layer in publishable_layers]
+ created_instances.append(instance)
+
+ for instance in created_instances:
+ # Produce diagnostic message for any graphical
+ # user interface interested in visualising it.
+ self.log.info("Found: \"%s\" " % instance.data["name"])
+ self.log.info("instance: {} ".format(instance.data))
+
+ def _get_existing_subset_names(self, context):
+ """Collect manually created instances from workfile.
+
+ Shouldn't be any as Webpublisher bypass publishing via Openpype, but
+ might be some if workfile published through OP is reused.
+ """
+ existing_subset_names = []
+ for instance in context:
+ if instance.data.get('publish'):
+ existing_subset_names.append(instance.data.get('subset'))
+
+ return existing_subset_names
+
+ def _create_instance(self, context, layer, family,
+ asset, subset, task_name):
+ instance = context.create_instance(layer.name)
+ instance.data["family"] = family
+ instance.data["publish"] = True
+ instance.data["asset"] = asset
+ instance.data["task"] = task_name
+ instance.data["subset"] = subset
+ instance.data["layer"] = layer
+ instance.data["families"] = []
+
+ return instance
+
+ def _resolve_mapping(self, layer):
+ """Matches 'layer' color code and name to mapping.
+
+ If both color code AND name regex is configured, BOTH must be valid
+ If layer matches to multiple mappings, only first is used!
+ """
+ family_list = []
+ family = None
+ subset_name_list = []
+ resolved_subset_template = None
+ for mapping in self.color_code_mapping:
+ if mapping["color_code"] and \
+ layer.color_code not in mapping["color_code"]:
+ continue
+
+ if mapping["layer_name_regex"] and \
+ not any(re.search(pattern, layer.name)
+ for pattern in mapping["layer_name_regex"]):
+ continue
+
+ family_list.append(mapping["family"])
+ subset_name_list.append(mapping["subset_template_name"])
+ if len(subset_name_list) > 1:
+ self.log.warning("Multiple mappings found for '{}'".
+ format(layer.name))
+ self.log.warning("Only first subset name template used!")
+ subset_name_list[:] = subset_name_list[0]
+
+ if len(family_list) > 1:
+ self.log.warning("Multiple mappings found for '{}'".
+ format(layer.name))
+ self.log.warning("Only first family used!")
+ family_list[:] = family_list[0]
+ if subset_name_list:
+ resolved_subset_template = subset_name_list.pop()
+ if family_list:
+ family = family_list.pop()
+
+ self.log.debug("resolved_family {}".format(family))
+ self.log.debug("resolved_subset_template {}".format(
+ resolved_subset_template))
+ return family, resolved_subset_template
+
+ def _clean_subset_name(self, stub, naming_conventions, subset, layer):
+ """Cleans invalid characters from subset name and layer name."""
+ if re.search(naming_conventions["invalid_chars"], subset):
+ subset = re.sub(
+ naming_conventions["invalid_chars"],
+ naming_conventions["replace_char"],
+ subset
+ )
+ layer_name = re.sub(
+ naming_conventions["invalid_chars"],
+ naming_conventions["replace_char"],
+ layer.clean_name
+ )
+ layer.name = layer_name
+ stub.rename_layer(layer.id, layer_name)
+
+ return subset
diff --git a/openpype/hosts/photoshop/plugins/publish/collect_current_file.py b/openpype/hosts/photoshop/plugins/publish/collect_current_file.py
index 4d4829555ec..5daf47c6ac3 100644
--- a/openpype/hosts/photoshop/plugins/publish/collect_current_file.py
+++ b/openpype/hosts/photoshop/plugins/publish/collect_current_file.py
@@ -2,7 +2,7 @@
import pyblish.api
-from avalon import photoshop
+from openpype.hosts.photoshop import api as photoshop
class CollectCurrentFile(pyblish.api.ContextPlugin):
diff --git a/openpype/hosts/photoshop/plugins/publish/collect_extension_version.py b/openpype/hosts/photoshop/plugins/publish/collect_extension_version.py
index f07ff0b0ff7..64c99b4fc1e 100644
--- a/openpype/hosts/photoshop/plugins/publish/collect_extension_version.py
+++ b/openpype/hosts/photoshop/plugins/publish/collect_extension_version.py
@@ -2,7 +2,7 @@
import re
import pyblish.api
-from avalon import photoshop
+from openpype.hosts.photoshop import api as photoshop
class CollectExtensionVersion(pyblish.api.ContextPlugin):
diff --git a/openpype/hosts/photoshop/plugins/publish/collect_instances.py b/openpype/hosts/photoshop/plugins/publish/collect_instances.py
index 5390df768b3..b466ec86879 100644
--- a/openpype/hosts/photoshop/plugins/publish/collect_instances.py
+++ b/openpype/hosts/photoshop/plugins/publish/collect_instances.py
@@ -1,61 +1,116 @@
+import pprint
+
import pyblish.api
-from avalon import photoshop
+from openpype.settings import get_project_settings
+from openpype.hosts.photoshop import api as photoshop
+from openpype.lib import prepare_template_data
+from openpype.pipeline import legacy_io
class CollectInstances(pyblish.api.ContextPlugin):
"""Gather instances by LayerSet and file metadata
- This collector takes into account assets that are associated with
- an LayerSet and marked with a unique identifier;
+ Collects publishable instances from file metadata or enhance
+ already collected by creator (family == "image").
+
+ If no image instances are explicitly created, it looks if there is value
+ in `flatten_subset_template` (configurable in Settings), in that case it
+ produces flatten image with all visible layers.
Identifier:
id (str): "pyblish.avalon.instance"
"""
- label = "Instances"
+ label = "Collect Instances"
order = pyblish.api.CollectorOrder
hosts = ["photoshop"]
families_mapping = {
"image": []
}
+ # configurable in Settings
+ flatten_subset_template = ""
def process(self, context):
+ instance_by_layer_id = {}
+ for instance in context:
+ if (
+ instance.data["family"] == "image" and
+ instance.data.get("members")):
+ layer_id = str(instance.data["members"][0])
+ instance_by_layer_id[layer_id] = instance
+
stub = photoshop.stub()
- layers = stub.get_layers()
+ layer_items = stub.get_layers()
layers_meta = stub.get_layers_metadata()
instance_names = []
- for layer in layers:
- layer_data = stub.read(layer, layers_meta)
+
+ all_layer_ids = []
+ for layer_item in layer_items:
+ layer_meta_data = stub.read(layer_item, layers_meta)
+ all_layer_ids.append(layer_item.id)
# Skip layers without metadata.
- if layer_data is None:
+ if layer_meta_data is None:
continue
# Skip containers.
- if "container" in layer_data["id"]:
+ if "container" in layer_meta_data["id"]:
+ continue
+
+ # active might not be in legacy meta
+ if not layer_meta_data.get("active", True):
continue
- # child_layers = [*layer.Layers]
- # self.log.debug("child_layers {}".format(child_layers))
- # if not child_layers:
- # self.log.info("%s skipped, it was empty." % layer.Name)
- # continue
+ instance = instance_by_layer_id.get(str(layer_item.id))
+ if instance is None:
+ instance = context.create_instance(layer_meta_data["subset"])
- instance = context.create_instance(layer_data["subset"])
- instance.append(layer)
- instance.data.update(layer_data)
+ instance.data["layer"] = layer_item
+ instance.data.update(layer_meta_data)
instance.data["families"] = self.families_mapping[
- layer_data["family"]
+ layer_meta_data["family"]
]
- instance.data["publish"] = layer.visible
- instance_names.append(layer_data["subset"])
+ instance.data["publish"] = layer_item.visible
+ instance_names.append(layer_meta_data["subset"])
# Produce diagnostic message for any graphical
# user interface interested in visualising it.
self.log.info("Found: \"%s\" " % instance.data["name"])
- self.log.info("instance: {} ".format(instance.data))
+ self.log.info("instance: {} ".format(
+ pprint.pformat(instance.data, indent=4)))
if len(instance_names) != len(set(instance_names)):
self.log.warning("Duplicate instances found. " +
"Remove unwanted via SubsetManager")
+
+ if len(instance_names) == 0 and self.flatten_subset_template:
+ project_name = context.data["projectEntity"]["name"]
+ variants = get_project_settings(project_name).get(
+ "photoshop", {}).get(
+ "create", {}).get(
+ "CreateImage", {}).get(
+ "defaults", [''])
+ family = "image"
+ task_name = legacy_io.Session["AVALON_TASK"]
+ asset_name = context.data["assetEntity"]["name"]
+
+ variant = context.data.get("variant") or variants[0]
+ fill_pairs = {
+ "variant": variant,
+ "family": family,
+ "task": task_name
+ }
+
+ subset = self.flatten_subset_template.format(
+ **prepare_template_data(fill_pairs))
+
+ instance = context.create_instance(subset)
+ instance.data["family"] = family
+ instance.data["asset"] = asset_name
+ instance.data["subset"] = subset
+ instance.data["ids"] = all_layer_ids
+ instance.data["families"] = self.families_mapping[family]
+ instance.data["publish"] = True
+
+ self.log.info("flatten instance: {} ".format(instance.data))
diff --git a/openpype/hosts/photoshop/plugins/publish/collect_published_version.py b/openpype/hosts/photoshop/plugins/publish/collect_published_version.py
new file mode 100644
index 00000000000..2502689e4bb
--- /dev/null
+++ b/openpype/hosts/photoshop/plugins/publish/collect_published_version.py
@@ -0,0 +1,55 @@
+"""Collects published version of workfile and increments it.
+
+For synchronization of published image and workfile version it is required
+to store workfile version from workfile file name in context.data["version"].
+In remote publishing this name is unreliable (artist might not follow naming
+convention etc.), last published workfile version for particular workfile
+subset is used instead.
+
+This plugin runs only in remote publishing (eg. Webpublisher).
+
+Requires:
+ context.data["assetEntity"]
+
+Provides:
+ context["version"] - incremented latest published workfile version
+"""
+
+import pyblish.api
+
+from openpype.client import get_last_version_by_subset_name
+
+
+class CollectPublishedVersion(pyblish.api.ContextPlugin):
+ """Collects published version of workfile and increments it."""
+
+ order = pyblish.api.CollectorOrder + 0.190
+ label = "Collect published version"
+ hosts = ["photoshop"]
+ targets = ["remotepublish"]
+
+ def process(self, context):
+ workfile_subset_name = None
+ for instance in context:
+ if instance.data["family"] == "workfile":
+ workfile_subset_name = instance.data["subset"]
+ break
+
+ if not workfile_subset_name:
+ self.log.warning("No workfile instance found, "
+ "synchronization of version will not work.")
+ return
+
+ project_name = context.data["projectName"]
+ asset_doc = context.data["assetEntity"]
+ asset_id = asset_doc["_id"]
+
+ version_doc = get_last_version_by_subset_name(project_name,
+ workfile_subset_name,
+ asset_id)
+ version_int = 1
+ if version_doc:
+ version_int += int(version_doc["name"])
+
+ self.log.debug(f"Setting {version_int} to context.")
+ context.data["version"] = version_int
diff --git a/openpype/hosts/photoshop/plugins/publish/collect_remote_instances.py b/openpype/hosts/photoshop/plugins/publish/collect_remote_instances.py
deleted file mode 100644
index c76e15484ea..00000000000
--- a/openpype/hosts/photoshop/plugins/publish/collect_remote_instances.py
+++ /dev/null
@@ -1,133 +0,0 @@
-import pyblish.api
-import os
-import re
-
-from avalon import photoshop
-from openpype.lib import prepare_template_data
-from openpype.lib.plugin_tools import parse_json
-
-
-class CollectRemoteInstances(pyblish.api.ContextPlugin):
- """Gather instances configured color code of a layer.
-
- Used in remote publishing when artists marks publishable layers by color-
- coding.
-
- Identifier:
- id (str): "pyblish.avalon.instance"
- """
- order = pyblish.api.CollectorOrder + 0.100
-
- label = "Instances"
- order = pyblish.api.CollectorOrder
- hosts = ["photoshop"]
- targets = ["remotepublish"]
-
- # configurable by Settings
- color_code_mapping = []
-
- def process(self, context):
- self.log.info("CollectRemoteInstances")
- self.log.info("mapping:: {}".format(self.color_code_mapping))
-
- # parse variant if used in webpublishing, comes from webpublisher batch
- batch_dir = os.environ.get("OPENPYPE_PUBLISH_DATA")
- variant = "Main"
- if batch_dir and os.path.exists(batch_dir):
- # TODO check if batch manifest is same as tasks manifests
- task_data = parse_json(os.path.join(batch_dir,
- "manifest.json"))
- if not task_data:
- raise ValueError(
- "Cannot parse batch meta in {} folder".format(batch_dir))
- variant = task_data["variant"]
-
- stub = photoshop.stub()
- layers = stub.get_layers()
-
- instance_names = []
- for layer in layers:
- self.log.info("Layer:: {}".format(layer))
- resolved_family, resolved_subset_template = self._resolve_mapping(
- layer
- )
- self.log.info("resolved_family {}".format(resolved_family))
- self.log.info("resolved_subset_template {}".format(
- resolved_subset_template))
-
- if not resolved_subset_template or not resolved_family:
- self.log.debug("!!! Not marked, skip")
- continue
-
- if layer.parents:
- self.log.debug("!!! Not a top layer, skip")
- continue
-
- instance = context.create_instance(layer.name)
- instance.append(layer)
- instance.data["family"] = resolved_family
- instance.data["publish"] = layer.visible
- instance.data["asset"] = context.data["assetEntity"]["name"]
- instance.data["task"] = context.data["taskType"]
-
- fill_pairs = {
- "variant": variant,
- "family": instance.data["family"],
- "task": instance.data["task"],
- "layer": layer.name
- }
- subset = resolved_subset_template.format(
- **prepare_template_data(fill_pairs))
- instance.data["subset"] = subset
-
- instance_names.append(layer.name)
-
- # Produce diagnostic message for any graphical
- # user interface interested in visualising it.
- self.log.info("Found: \"%s\" " % instance.data["name"])
- self.log.info("instance: {} ".format(instance.data))
-
- if len(instance_names) != len(set(instance_names)):
- self.log.warning("Duplicate instances found. " +
- "Remove unwanted via SubsetManager")
-
- def _resolve_mapping(self, layer):
- """Matches 'layer' color code and name to mapping.
-
- If both color code AND name regex is configured, BOTH must be valid
- If layer matches to multiple mappings, only first is used!
- """
- family_list = []
- family = None
- subset_name_list = []
- resolved_subset_template = None
- for mapping in self.color_code_mapping:
- if mapping["color_code"] and \
- layer.color_code not in mapping["color_code"]:
- continue
-
- if mapping["layer_name_regex"] and \
- not any(re.search(pattern, layer.name)
- for pattern in mapping["layer_name_regex"]):
- continue
-
- family_list.append(mapping["family"])
- subset_name_list.append(mapping["subset_template_name"])
-
- if len(subset_name_list) > 1:
- self.log.warning("Multiple mappings found for '{}'".
- format(layer.name))
- self.log.warning("Only first subset name template used!")
- subset_name_list[:] = subset_name_list[0]
-
- if len(family_list) > 1:
- self.log.warning("Multiple mappings found for '{}'".
- format(layer.name))
- self.log.warning("Only first family used!")
- family_list[:] = family_list[0]
- if subset_name_list:
- resolved_subset_template = subset_name_list.pop()
- if family_list:
- family = family_list.pop()
-
- return family, resolved_subset_template
diff --git a/openpype/hosts/photoshop/plugins/publish/collect_review.py b/openpype/hosts/photoshop/plugins/publish/collect_review.py
index 5ab48b76da3..7f395b46d7a 100644
--- a/openpype/hosts/photoshop/plugins/publish/collect_review.py
+++ b/openpype/hosts/photoshop/plugins/publish/collect_review.py
@@ -1,30 +1,49 @@
+"""
+Requires:
+ None
+
+Provides:
+ instance -> family ("review")
+"""
+
import os
import pyblish.api
+from openpype.pipeline.create import get_subset_name
+
class CollectReview(pyblish.api.ContextPlugin):
- """Gather the active document as review instance."""
+ """Gather the active document as review instance.
+ Triggers once even if no 'image' is published as by defaults it creates
+ flatten image from a workfile.
+ """
+
+ label = "Collect Review"
label = "Review"
- order = pyblish.api.CollectorOrder
hosts = ["photoshop"]
+ order = pyblish.api.CollectorOrder + 0.1
def process(self, context):
family = "review"
- task = os.getenv("AVALON_TASK", None)
- subset = family + task.capitalize()
-
- file_path = context.data["currentFile"]
- base_name = os.path.basename(file_path)
+ subset = get_subset_name(
+ family,
+ context.data.get("variant", ''),
+ context.data["anatomyData"]["task"]["name"],
+ context.data["assetEntity"],
+ context.data["anatomyData"]["project"]["name"],
+ host_name=context.data["hostName"],
+ project_settings=context.data["project_settings"]
+ )
instance = context.create_instance(subset)
instance.data.update({
"subset": subset,
- "label": base_name,
- "name": base_name,
+ "label": subset,
+ "name": subset,
"family": family,
- "families": ["ftrack"],
+ "families": [],
"representations": [],
"asset": os.environ["AVALON_ASSET"]
})
diff --git a/openpype/hosts/photoshop/plugins/publish/collect_version.py b/openpype/hosts/photoshop/plugins/publish/collect_version.py
new file mode 100644
index 00000000000..aff9f13bfb5
--- /dev/null
+++ b/openpype/hosts/photoshop/plugins/publish/collect_version.py
@@ -0,0 +1,24 @@
+import pyblish.api
+
+
+class CollectVersion(pyblish.api.InstancePlugin):
+ """Collect version for publishable instances.
+
+ Used to synchronize version from workfile to all publishable instances:
+ - image (manually created or color coded)
+ - review
+
+ Dev comment:
+ Explicit collector created to control this from single place and not from
+ 3 different.
+ """
+ order = pyblish.api.CollectorOrder + 0.200
+ label = 'Collect Version'
+
+ hosts = ["photoshop"]
+ families = ["image", "review"]
+
+ def process(self, instance):
+ workfile_version = instance.context.data["version"]
+ self.log.debug(f"Applying version {workfile_version}")
+ instance.data["version"] = workfile_version
diff --git a/openpype/hosts/photoshop/plugins/publish/collect_workfile.py b/openpype/hosts/photoshop/plugins/publish/collect_workfile.py
index 88817c39690..9a5aad5569b 100644
--- a/openpype/hosts/photoshop/plugins/publish/collect_workfile.py
+++ b/openpype/hosts/photoshop/plugins/publish/collect_workfile.py
@@ -1,5 +1,7 @@
-import pyblish.api
import os
+import pyblish.api
+
+from openpype.pipeline.create import get_subset_name
class CollectWorkfile(pyblish.api.ContextPlugin):
@@ -9,26 +11,47 @@ class CollectWorkfile(pyblish.api.ContextPlugin):
label = "Collect Workfile"
hosts = ["photoshop"]
+ default_variant = "Main"
+
def process(self, context):
+ existing_instance = None
+ for instance in context:
+ if instance.data["family"] == "workfile":
+ self.log.debug("Workfile instance found, won't create new")
+ existing_instance = instance
+ break
+
family = "workfile"
- task = os.getenv("AVALON_TASK", None)
- subset = family + task.capitalize()
+ # context.data["variant"] might come only from collect_batch_data
+ variant = context.data.get("variant") or self.default_variant
+ subset = get_subset_name(
+ family,
+ variant,
+ context.data["anatomyData"]["task"]["name"],
+ context.data["assetEntity"],
+ context.data["anatomyData"]["project"]["name"],
+ host_name=context.data["hostName"],
+ project_settings=context.data["project_settings"]
+ )
file_path = context.data["currentFile"]
staging_dir = os.path.dirname(file_path)
base_name = os.path.basename(file_path)
# Create instance
- instance = context.create_instance(subset)
- instance.data.update({
- "subset": subset,
- "label": base_name,
- "name": base_name,
- "family": family,
- "families": [],
- "representations": [],
- "asset": os.environ["AVALON_ASSET"]
- })
+ if existing_instance is None:
+ instance = context.create_instance(subset)
+ instance.data.update({
+ "subset": subset,
+ "label": base_name,
+ "name": base_name,
+ "family": family,
+ "families": [],
+ "representations": [],
+ "asset": os.environ["AVALON_ASSET"]
+ })
+ else:
+ instance = existing_instance
# creating representation
_, ext = os.path.splitext(file_path)
diff --git a/openpype/hosts/photoshop/plugins/publish/extract_image.py b/openpype/hosts/photoshop/plugins/publish/extract_image.py
index ae9892e2903..cdb28c742df 100644
--- a/openpype/hosts/photoshop/plugins/publish/extract_image.py
+++ b/openpype/hosts/photoshop/plugins/publish/extract_image.py
@@ -1,60 +1,99 @@
import os
-import openpype.api
-from avalon import photoshop
+import pyblish.api
+from openpype.pipeline import publish
+from openpype.hosts.photoshop import api as photoshop
-class ExtractImage(openpype.api.Extractor):
- """Produce a flattened image file from instance
+class ExtractImage(pyblish.api.ContextPlugin):
+ """Extract all layers (groups) marked for publish.
- This plug-in takes into account only the layers in the group.
+ Usually publishable instance is created as a wrapper of layer(s). For each
+ publishable instance so many images as there is 'formats' is created.
+
+ Logic tries to hide/unhide layers minimum times.
+
+ Called once for all publishable instances.
"""
+ order = publish.Extractor.order - 0.48
label = "Extract Image"
hosts = ["photoshop"]
+
families = ["image", "background"]
formats = ["png", "jpg"]
- def process(self, instance):
+ def process(self, context):
+ stub = photoshop.stub()
+ hidden_layer_ids = set()
- staging_dir = self.staging_dir(instance)
- self.log.info("Outputting image to {}".format(staging_dir))
+ all_layers = stub.get_layers()
+ for layer in all_layers:
+ if not layer.visible:
+ hidden_layer_ids.add(layer.id)
+ stub.hide_all_others_layers_ids([], layers=all_layers)
- # Perform extraction
- stub = photoshop.stub()
- files = {}
with photoshop.maintained_selection():
- self.log.info("Extracting %s" % str(list(instance)))
- with photoshop.maintained_visibility():
- # Hide all other layers.
- extract_ids = set([ll.id for ll in stub.
- get_layers_in_layers([instance[0]])])
-
- for layer in stub.get_layers():
- # limit unnecessary calls to client
- if layer.visible and layer.id not in extract_ids:
- stub.set_visible(layer.id, False)
-
- file_basename = os.path.splitext(
- stub.get_active_document_name()
- )[0]
- for extension in self.formats:
- _filename = "{}.{}".format(file_basename, extension)
- files[extension] = _filename
-
- full_filename = os.path.join(staging_dir, _filename)
- stub.saveAs(full_filename, extension, True)
- self.log.info(f"Extracted: {extension}")
-
- representations = []
- for extension, filename in files.items():
- representations.append({
- "name": extension,
- "ext": extension,
- "files": filename,
- "stagingDir": staging_dir
- })
- instance.data["representations"] = representations
- instance.data["stagingDir"] = staging_dir
-
- self.log.info(f"Extracted {instance} to {staging_dir}")
+ with photoshop.maintained_visibility(layers=all_layers):
+ for instance in context:
+ if instance.data["family"] not in self.families:
+ continue
+
+ staging_dir = self.staging_dir(instance)
+ self.log.info("Outputting image to {}".format(staging_dir))
+
+ # Perform extraction
+ files = {}
+ ids = set()
+ layer = instance.data.get("layer")
+ if layer:
+ ids.add(layer.id)
+ add_ids = instance.data.pop("ids", None)
+ if add_ids:
+ ids.update(set(add_ids))
+ extract_ids = set([ll.id for ll in stub.
+ get_layers_in_layers_ids(ids, all_layers)
+ if ll.id not in hidden_layer_ids])
+
+ for extracted_id in extract_ids:
+ stub.set_visible(extracted_id, True)
+
+ file_basename = os.path.splitext(
+ stub.get_active_document_name()
+ )[0]
+ for extension in self.formats:
+ _filename = "{}.{}".format(file_basename,
+ extension)
+ files[extension] = _filename
+
+ full_filename = os.path.join(staging_dir,
+ _filename)
+ stub.saveAs(full_filename, extension, True)
+ self.log.info(f"Extracted: {extension}")
+
+ representations = []
+ for extension, filename in files.items():
+ representations.append({
+ "name": extension,
+ "ext": extension,
+ "files": filename,
+ "stagingDir": staging_dir
+ })
+ instance.data["representations"] = representations
+ instance.data["stagingDir"] = staging_dir
+
+ self.log.info(f"Extracted {instance} to {staging_dir}")
+
+ for extracted_id in extract_ids:
+ stub.set_visible(extracted_id, False)
+
+ def staging_dir(self, instance):
+ """Provide a temporary directory in which to store extracted files
+
+ Upon calling this method the staging directory is stored inside
+ the instance.data['stagingDir']
+ """
+
+ from openpype.pipeline.publish import get_instance_staging_dir
+
+ return get_instance_staging_dir(instance)
diff --git a/openpype/hosts/photoshop/plugins/publish/extract_review.py b/openpype/hosts/photoshop/plugins/publish/extract_review.py
index 8c4d05b282a..e5fee311f83 100644
--- a/openpype/hosts/photoshop/plugins/publish/extract_review.py
+++ b/openpype/hosts/photoshop/plugins/publish/extract_review.py
@@ -1,16 +1,28 @@
import os
+import shutil
+from PIL import Image
-import openpype.api
-import openpype.lib
-from avalon import photoshop
+from openpype.lib import (
+ run_subprocess,
+ get_ffmpeg_tool_path,
+)
+from openpype.pipeline import publish
+from openpype.hosts.photoshop import api as photoshop
-class ExtractReview(openpype.api.Extractor):
+class ExtractReview(publish.Extractor):
"""
- Produce a flattened image file from all 'image' instances.
+ Produce a flattened or sequence image files from all 'image' instances.
If no 'image' instance is created, it produces flattened image from
all visible layers.
+
+ It creates review, thumbnail and mov representations.
+
+ 'review' family could be used in other steps as a reference, as it
+ contains flattened image by default. (Eg. artist could load this
+ review as a single item and see full image. In most cases 'image'
+ family is separated by layers to better usage in animation or comp.)
"""
label = "Extract Review"
@@ -20,59 +32,71 @@ class ExtractReview(openpype.api.Extractor):
# Extract Options
jpg_options = None
mov_options = None
+ make_image_sequence = None
+ max_downscale_size = 8192
def process(self, instance):
staging_dir = self.staging_dir(instance)
self.log.info("Outputting image to {}".format(staging_dir))
+ fps = instance.data.get("fps", 25)
stub = photoshop.stub()
+ self.output_seq_filename = os.path.splitext(
+ stub.get_active_document_name())[0] + ".%04d.jpg"
+
+ layers = self._get_layers_from_image_instances(instance)
+ self.log.info("Layers image instance found: {}".format(layers))
+
+ if self.make_image_sequence and len(layers) > 1:
+ self.log.info("Extract layers to image sequence.")
+ img_list = self._saves_sequences_layers(staging_dir, layers)
+
+ instance.data["representations"].append({
+ "name": "jpg",
+ "ext": "jpg",
+ "files": img_list,
+ "frameStart": 0,
+ "frameEnd": len(img_list),
+ "fps": fps,
+ "stagingDir": staging_dir,
+ "tags": self.jpg_options['tags'],
+ })
+ processed_img_names = img_list
+ else:
+ self.log.info("Extract layers to flatten image.")
+ img_list = self._saves_flattened_layers(staging_dir, layers)
+
+ instance.data["representations"].append({
+ "name": "jpg",
+ "ext": "jpg",
+ "files": img_list, # cannot be [] for single frame
+ "stagingDir": staging_dir,
+ "tags": self.jpg_options['tags']
+ })
+ processed_img_names = [img_list]
+
+ ffmpeg_path = get_ffmpeg_tool_path("ffmpeg")
- layers = []
- for image_instance in instance.context:
- if image_instance.data["family"] != "image":
- continue
- layers.append(image_instance[0])
-
- # Perform extraction
- output_image = "{}.jpg".format(
- os.path.splitext(stub.get_active_document_name())[0]
- )
- output_image_path = os.path.join(staging_dir, output_image)
- with photoshop.maintained_visibility():
- if layers:
- # Hide all other layers.
- extract_ids = set([ll.id for ll in stub.
- get_layers_in_layers(layers)])
- self.log.debug("extract_ids {}".format(extract_ids))
- for layer in stub.get_layers():
- # limit unnecessary calls to client
- if layer.visible and layer.id not in extract_ids:
- stub.set_visible(layer.id, False)
-
- stub.saveAs(output_image_path, 'jpg', True)
-
- ffmpeg_path = openpype.lib.get_ffmpeg_tool_path("ffmpeg")
-
- instance.data["representations"].append({
- "name": "jpg",
- "ext": "jpg",
- "files": output_image,
- "stagingDir": staging_dir,
- "tags": self.jpg_options['tags']
- })
instance.data["stagingDir"] = staging_dir
- # Generate thumbnail.
+ source_files_pattern = os.path.join(staging_dir,
+ self.output_seq_filename)
+ source_files_pattern = self._check_and_resize(processed_img_names,
+ source_files_pattern,
+ staging_dir)
+ # Generate thumbnail
thumbnail_path = os.path.join(staging_dir, "thumbnail.jpg")
+ self.log.info(f"Generate thumbnail {thumbnail_path}")
args = [
ffmpeg_path,
"-y",
- "-i", output_image_path,
+ "-i", source_files_pattern,
"-vf", "scale=300:-1",
"-vframes", "1",
thumbnail_path
]
- output = openpype.lib.run_subprocess(args)
+ self.log.debug("thumbnail args:: {}".format(args))
+ output = run_subprocess(args)
instance.data["representations"].append({
"name": "thumbnail",
@@ -81,17 +105,21 @@ def process(self, instance):
"stagingDir": staging_dir,
"tags": ["thumbnail"]
})
+
# Generate mov.
mov_path = os.path.join(staging_dir, "review.mov")
+ self.log.info(f"Generate mov review: {mov_path}")
+ img_number = len(img_list)
args = [
ffmpeg_path,
"-y",
- "-i", output_image_path,
+ "-i", source_files_pattern,
"-vf", "pad=ceil(iw/2)*2:ceil(ih/2)*2",
- "-vframes", "1",
+ "-vframes", str(img_number),
mov_path
]
- output = openpype.lib.run_subprocess(args)
+ self.log.debug("mov args:: {}".format(args))
+ output = run_subprocess(args)
self.log.debug(output)
instance.data["representations"].append({
"name": "mov",
@@ -99,15 +127,117 @@ def process(self, instance):
"files": os.path.basename(mov_path),
"stagingDir": staging_dir,
"frameStart": 1,
- "frameEnd": 1,
- "fps": 25,
+ "frameEnd": img_number,
+ "fps": fps,
"preview": True,
"tags": self.mov_options['tags']
})
# Required for extract_review plugin (L222 onwards).
instance.data["frameStart"] = 1
- instance.data["frameEnd"] = 1
+ instance.data["frameEnd"] = img_number
instance.data["fps"] = 25
self.log.info(f"Extracted {instance} to {staging_dir}")
+
+ def _check_and_resize(self, processed_img_names, source_files_pattern,
+ staging_dir):
+ """Check if saved image could be used in ffmpeg.
+
+ Ffmpeg has max size 16384x16384. Saved image(s) must be resized to be
+ used as a source for thumbnail or review mov.
+ """
+ Image.MAX_IMAGE_PIXELS = None
+ first_url = os.path.join(staging_dir, processed_img_names[0])
+ with Image.open(first_url) as im:
+ width, height = im.size
+
+ if width > self.max_downscale_size or height > self.max_downscale_size:
+ resized_dir = os.path.join(staging_dir, "resized")
+ os.mkdir(resized_dir)
+ source_files_pattern = os.path.join(resized_dir,
+ self.output_seq_filename)
+ for file_name in processed_img_names:
+ source_url = os.path.join(staging_dir, file_name)
+ with Image.open(source_url) as res_img:
+ # 'thumbnail' automatically keeps aspect ratio
+ res_img.thumbnail((self.max_downscale_size,
+ self.max_downscale_size),
+ Image.ANTIALIAS)
+ res_img.save(os.path.join(resized_dir, file_name))
+
+ return source_files_pattern
+
+ def _get_image_path_from_instances(self, instance):
+ img_list = []
+
+ for instance in sorted(instance.context):
+ if instance.data["family"] != "image":
+ continue
+
+ for rep in instance.data["representations"]:
+ img_path = os.path.join(
+ rep["stagingDir"],
+ rep["files"]
+ )
+ img_list.append(img_path)
+
+ return img_list
+
+ def _copy_image_to_staging_dir(self, staging_dir, img_list):
+ copy_files = []
+ for i, img_src in enumerate(img_list):
+ img_filename = self.output_seq_filename % i
+ img_dst = os.path.join(staging_dir, img_filename)
+
+ self.log.debug(
+ "Copying file .. {} -> {}".format(img_src, img_dst)
+ )
+ shutil.copy(img_src, img_dst)
+ copy_files.append(img_filename)
+
+ return copy_files
+
+ def _get_layers_from_image_instances(self, instance):
+ layers = []
+ for image_instance in instance.context:
+ if image_instance.data["family"] != "image":
+ continue
+ if not image_instance.data.get("layer"):
+ # dummy instance for flatten image
+ continue
+ layers.append(image_instance.data.get("layer"))
+
+ return sorted(layers)
+
+ def _saves_flattened_layers(self, staging_dir, layers):
+ img_filename = self.output_seq_filename % 0
+ output_image_path = os.path.join(staging_dir, img_filename)
+ stub = photoshop.stub()
+
+ with photoshop.maintained_visibility():
+ self.log.info("Extracting {}".format(layers))
+ if layers:
+ stub.hide_all_others_layers(layers)
+
+ stub.saveAs(output_image_path, 'jpg', True)
+
+ return img_filename
+
+ def _saves_sequences_layers(self, staging_dir, layers):
+ stub = photoshop.stub()
+
+ list_img_filename = []
+ with photoshop.maintained_visibility():
+ for i, layer in enumerate(layers):
+ self.log.info("Extracting {}".format(layer))
+
+ img_filename = self.output_seq_filename % i
+ output_image_path = os.path.join(staging_dir, img_filename)
+ list_img_filename.append(img_filename)
+
+ with photoshop.maintained_visibility():
+ stub.hide_all_others_layers([layer])
+ stub.saveAs(output_image_path, 'jpg', True)
+
+ return list_img_filename
diff --git a/openpype/hosts/photoshop/plugins/publish/extract_save_scene.py b/openpype/hosts/photoshop/plugins/publish/extract_save_scene.py
index 0180640c908..aa900fec9fa 100644
--- a/openpype/hosts/photoshop/plugins/publish/extract_save_scene.py
+++ b/openpype/hosts/photoshop/plugins/publish/extract_save_scene.py
@@ -1,11 +1,11 @@
-import openpype.api
-from avalon import photoshop
+from openpype.pipeline import publish
+from openpype.hosts.photoshop import api as photoshop
-class ExtractSaveScene(openpype.api.Extractor):
+class ExtractSaveScene(publish.Extractor):
"""Save scene before extraction."""
- order = openpype.api.Extractor.order - 0.49
+ order = publish.Extractor.order - 0.49
label = "Extract Save Scene"
hosts = ["photoshop"]
families = ["workfile"]
diff --git a/openpype/hosts/photoshop/plugins/publish/help/validate_naming.xml b/openpype/hosts/photoshop/plugins/publish/help/validate_naming.xml
new file mode 100644
index 00000000000..5a1e266748b
--- /dev/null
+++ b/openpype/hosts/photoshop/plugins/publish/help/validate_naming.xml
@@ -0,0 +1,21 @@
+
+
+
+Subset name
+
+## Invalid subset or layer name
+
+Subset or layer name cannot contain specific characters (spaces etc) which could cause issue when subset name is used in a published file name.
+ {msg}
+
+### How to repair?
+
+You can fix this with "repair" button on the right.
+
+
+### __Detailed Info__ (optional)
+
+Not all characters are available in a file names on all OS. Wrong characters could be configured in Settings.
+
+
+
\ No newline at end of file
diff --git a/openpype/hosts/photoshop/plugins/publish/help/validate_unique_subsets.xml b/openpype/hosts/photoshop/plugins/publish/help/validate_unique_subsets.xml
new file mode 100644
index 00000000000..4b479731935
--- /dev/null
+++ b/openpype/hosts/photoshop/plugins/publish/help/validate_unique_subsets.xml
@@ -0,0 +1,14 @@
+
+
+
+Subset not unique
+
+## Non unique subset name found
+
+ Non unique subset names: '{non_unique}'
+### How to repair?
+
+Remove offending instance, rename it to have unique name. Maybe layer name wasn't used for multiple instances?
+
+
+
\ No newline at end of file
diff --git a/openpype/hosts/photoshop/plugins/publish/increment_workfile.py b/openpype/hosts/photoshop/plugins/publish/increment_workfile.py
index 709fb988fca..665dd67fc5e 100644
--- a/openpype/hosts/photoshop/plugins/publish/increment_workfile.py
+++ b/openpype/hosts/photoshop/plugins/publish/increment_workfile.py
@@ -1,9 +1,9 @@
import os
import pyblish.api
-from openpype.action import get_errored_plugins_from_data
+from openpype.pipeline.publish import get_errored_plugins_from_context
from openpype.lib import version_up
-from avalon import photoshop
+from openpype.hosts.photoshop import api as photoshop
class IncrementWorkfile(pyblish.api.InstancePlugin):
@@ -19,7 +19,7 @@ class IncrementWorkfile(pyblish.api.InstancePlugin):
optional = True
def process(self, instance):
- errored_plugins = get_errored_plugins_from_data(instance.context)
+ errored_plugins = get_errored_plugins_from_context(instance.context)
if errored_plugins:
raise RuntimeError(
"Skipping incrementing current file because publishing failed."
diff --git a/openpype/hosts/photoshop/plugins/publish/validate_instance_asset.py b/openpype/hosts/photoshop/plugins/publish/validate_instance_asset.py
index 4dc1972074c..2609f7a8cf8 100644
--- a/openpype/hosts/photoshop/plugins/publish/validate_instance_asset.py
+++ b/openpype/hosts/photoshop/plugins/publish/validate_instance_asset.py
@@ -1,7 +1,8 @@
-from avalon import api
import pyblish.api
-import openpype.api
-from avalon import photoshop
+
+from openpype.pipeline import legacy_io
+from openpype.pipeline.publish import ValidateContentsOrder
+from openpype.hosts.photoshop import api as photoshop
class ValidateInstanceAssetRepair(pyblish.api.Action):
@@ -26,7 +27,7 @@ def process(self, context, plugin):
for instance in instances:
data = stub.read(instance[0])
- data["asset"] = api.Session["AVALON_ASSET"]
+ data["asset"] = legacy_io.Session["AVALON_ASSET"]
stub.imprint(instance[0], data)
@@ -44,11 +45,11 @@ class ValidateInstanceAsset(pyblish.api.InstancePlugin):
label = "Validate Instance Asset"
hosts = ["photoshop"]
actions = [ValidateInstanceAssetRepair]
- order = openpype.api.ValidateContentsOrder
+ order = ValidateContentsOrder
def process(self, instance):
instance_asset = instance.data["asset"]
- current_asset = api.Session["AVALON_ASSET"]
+ current_asset = legacy_io.Session["AVALON_ASSET"]
msg = (
f"Instance asset {instance_asset} is not the same "
f"as current context {current_asset}. PLEASE DO:\n"
diff --git a/openpype/hosts/photoshop/plugins/publish/validate_naming.py b/openpype/hosts/photoshop/plugins/publish/validate_naming.py
index 1635096f4b2..0665aff9d06 100644
--- a/openpype/hosts/photoshop/plugins/publish/validate_naming.py
+++ b/openpype/hosts/photoshop/plugins/publish/validate_naming.py
@@ -1,8 +1,13 @@
import re
import pyblish.api
-import openpype.api
-from avalon import photoshop
+
+from openpype.hosts.photoshop import api as photoshop
+from openpype.pipeline.create import SUBSET_NAME_ALLOWED_SYMBOLS
+from openpype.pipeline.publish import (
+ ValidateContentsOrder,
+ PublishXmlValidationError,
+)
class ValidateNamingRepair(pyblish.api.Action):
@@ -22,32 +27,42 @@ def process(self, context, plugin):
failed.append(result["instance"])
invalid_chars, replace_char = plugin.get_replace_chars()
- self.log.info("{} --- {}".format(invalid_chars, replace_char))
+ self.log.debug("{} --- {}".format(invalid_chars, replace_char))
# Apply pyblish.logic to get the instances for the plug-in
instances = pyblish.api.instances_by_plugin(failed, plugin)
stub = photoshop.stub()
for instance in instances:
- self.log.info("validate_naming instance {}".format(instance))
- metadata = stub.read(instance[0])
- self.log.info("metadata instance {}".format(metadata))
- layer_name = None
- if metadata.get("uuid"):
- layer_data = stub.get_layer(metadata["uuid"])
- self.log.info("layer_data {}".format(layer_data))
- if layer_data:
- layer_name = re.sub(invalid_chars,
- replace_char,
- layer_data.name)
-
- stub.rename_layer(instance.data["uuid"], layer_name)
+ self.log.debug("validate_naming instance {}".format(instance))
+ current_layer_state = stub.get_layer(instance.data["layer"].id)
+ self.log.debug("current_layer{}".format(current_layer_state))
+
+ layer_meta = stub.read(current_layer_state)
+ instance_id = (layer_meta.get("instance_id") or
+ layer_meta.get("uuid"))
+ if not instance_id:
+ self.log.warning("Unable to repair, cannot find layer")
+ continue
+
+ layer_name = re.sub(invalid_chars,
+ replace_char,
+ current_layer_state.clean_name)
+ layer_name = stub.PUBLISH_ICON + layer_name
+
+ stub.rename_layer(current_layer_state.id, layer_name)
subset_name = re.sub(invalid_chars, replace_char,
- instance.data["name"])
+ instance.data["subset"])
- instance[0].Name = layer_name or subset_name
- metadata["subset"] = subset_name
- stub.imprint(instance[0], metadata)
+ # format from Tool Creator
+ subset_name = re.sub(
+ "[^{}]+".format(SUBSET_NAME_ALLOWED_SYMBOLS),
+ "",
+ subset_name
+ )
+
+ layer_meta["subset"] = subset_name
+ stub.imprint(instance_id, layer_meta)
return True
@@ -60,7 +75,7 @@ class ValidateNaming(pyblish.api.InstancePlugin):
label = "Validate Naming"
hosts = ["photoshop"]
- order = openpype.api.ValidateContentsOrder
+ order = ValidateContentsOrder
families = ["image"]
actions = [ValidateNamingRepair]
@@ -70,13 +85,24 @@ class ValidateNaming(pyblish.api.InstancePlugin):
def process(self, instance):
help_msg = ' Use Repair action (A) in Pyblish to fix it.'
- msg = "Name \"{}\" is not allowed.{}".format(instance.data["name"],
- help_msg)
- assert not re.search(self.invalid_chars, instance.data["name"]), msg
+
+ layer = instance.data.get("layer")
+ if layer:
+ msg = "Name \"{}\" is not allowed.{}".format(layer.clean_name,
+ help_msg)
+
+ formatting_data = {"msg": msg}
+ if re.search(self.invalid_chars, layer.clean_name):
+ raise PublishXmlValidationError(self, msg,
+ formatting_data=formatting_data
+ )
msg = "Subset \"{}\" is not allowed.{}".format(instance.data["subset"],
help_msg)
- assert not re.search(self.invalid_chars, instance.data["subset"]), msg
+ formatting_data = {"msg": msg}
+ if re.search(self.invalid_chars, instance.data["subset"]):
+ raise PublishXmlValidationError(self, msg,
+ formatting_data=formatting_data)
@classmethod
def get_replace_chars(cls):
diff --git a/openpype/hosts/photoshop/plugins/publish/validate_unique_subsets.py b/openpype/hosts/photoshop/plugins/publish/validate_unique_subsets.py
index 15ae5fbcea2..78e84729cea 100644
--- a/openpype/hosts/photoshop/plugins/publish/validate_unique_subsets.py
+++ b/openpype/hosts/photoshop/plugins/publish/validate_unique_subsets.py
@@ -1,5 +1,9 @@
+import collections
import pyblish.api
-import openpype.api
+from openpype.pipeline.publish import (
+ ValidateContentsOrder,
+ PublishXmlValidationError,
+)
class ValidateSubsetUniqueness(pyblish.api.ContextPlugin):
@@ -9,18 +13,27 @@ class ValidateSubsetUniqueness(pyblish.api.ContextPlugin):
label = "Validate Subset Uniqueness"
hosts = ["photoshop"]
- order = openpype.api.ValidateContentsOrder
+ order = ValidateContentsOrder
families = ["image"]
def process(self, context):
subset_names = []
for instance in context:
+ self.log.info("instance:: {}".format(instance.data))
if instance.data.get('publish'):
subset_names.append(instance.data.get('subset'))
- msg = (
- "Instance subset names are not unique. " +
- "Remove duplicates via SubsetManager."
- )
- assert len(subset_names) == len(set(subset_names)), msg
+ non_unique = \
+ [item
+ for item, count in collections.Counter(subset_names).items()
+ if count > 1]
+ msg = ("Instance subset names {} are not unique. ".format(non_unique) +
+ "Remove duplicates via SubsetManager.")
+ formatting_data = {
+ "non_unique": ",".join(non_unique)
+ }
+
+ if non_unique:
+ raise PublishXmlValidationError(self, msg,
+ formatting_data=formatting_data)
diff --git a/openpype/hosts/resolve/README.markdown b/openpype/hosts/resolve/README.markdown
index 50664fbd219..8c9f72fb0c1 100644
--- a/openpype/hosts/resolve/README.markdown
+++ b/openpype/hosts/resolve/README.markdown
@@ -4,10 +4,10 @@
- add absolute path to ffmpeg into openpype settings
- install Python 3.6 into `%LOCALAPPDATA%/Programs/Python/Python36` (only respected path by Resolve)
-- install OpenTimelineIO for 3.6 `%LOCALAPPDATA%\Programs\Python\Python36\python.exe -m pip install git+https://github.com/PixarAnimationStudios/OpenTimelineIO.git@5aa24fbe89d615448876948fe4b4900455c9a3e8` and move builded files from `%LOCALAPPDATA%/Programs/Python/Python36/Lib/site-packages/opentimelineio/cxx-libs/bin and lib` to `%LOCALAPPDATA%/Programs/Python/Python36/Lib/site-packages/opentimelineio/`. I was building it on Win10 machine with Visual Studio Community 2019 and
+- install OpenTimelineIO for 3.6 `%LOCALAPPDATA%\Programs\Python\Python36\python.exe -m pip install git+https://github.com/PixarAnimationStudios/OpenTimelineIO.git@5aa24fbe89d615448876948fe4b4900455c9a3e8` and move built files from `%LOCALAPPDATA%/Programs/Python/Python36/Lib/site-packages/opentimelineio/cxx-libs/bin and lib` to `%LOCALAPPDATA%/Programs/Python/Python36/Lib/site-packages/opentimelineio/`. I was building it on Win10 machine with Visual Studio Community 2019 and
 with installed CMake in PATH.
- install PySide2 for 3.6 `%LOCALAPPDATA%\Programs\Python\Python36\python.exe -m pip install PySide2`
-- make sure Resovle Fusion (Fusion Tab/menu/Fusion/Fusion Setings) is set to Python 3.6
+- make sure Resolve Fusion (Fusion Tab/menu/Fusion/Fusion Settings) is set to Python 3.6
#### Editorial setup
diff --git a/openpype/hosts/resolve/RESOLVE_API_README_v16.2.0_up.txt b/openpype/hosts/resolve/RESOLVE_API_README_v16.2.0_up.txt
index a24a053cd77..f1b8b81a710 100644
--- a/openpype/hosts/resolve/RESOLVE_API_README_v16.2.0_up.txt
+++ b/openpype/hosts/resolve/RESOLVE_API_README_v16.2.0_up.txt
@@ -366,7 +366,7 @@ TimelineItem
DeleteTakeByIndex(idx) --> Bool # Deletes a take by index, 1 <= idx <= number of takes.
SelectTakeByIndex(idx) --> Bool # Selects a take by index, 1 <= idx <= number of takes.
FinalizeTake() --> Bool # Finalizes take selection.
- CopyGrades([tgtTimelineItems]) --> Bool # Copies the current grade to all the items in tgtTimelineItems list. Returns True on success and False if any error occured.
+ CopyGrades([tgtTimelineItems]) --> Bool # Copies the current grade to all the items in tgtTimelineItems list. Returns True on success and False if any error occurred.
List and Dict Data Structures
diff --git a/openpype/hosts/resolve/__init__.py b/openpype/hosts/resolve/__init__.py
index 3e49ce3b9b8..b4a994bbaa0 100644
--- a/openpype/hosts/resolve/__init__.py
+++ b/openpype/hosts/resolve/__init__.py
@@ -1,129 +1,6 @@
-from .api.utils import (
- setup,
- get_resolve_module
-)
-
-from .api.pipeline import (
- install,
- uninstall,
- ls,
- containerise,
- update_container,
- publish,
- launch_workfiles_app,
- maintained_selection,
- remove_instance,
- list_instances
-)
-
-from .api.lib import (
- maintain_current_timeline,
- publish_clip_color,
- get_project_manager,
- get_current_project,
- get_current_timeline,
- create_bin,
- get_media_pool_item,
- create_media_pool_item,
- create_timeline_item,
- get_timeline_item,
- get_video_track_names,
- get_current_timeline_items,
- get_pype_timeline_item_by_name,
- get_timeline_item_pype_tag,
- set_timeline_item_pype_tag,
- imprint,
- set_publish_attribute,
- get_publish_attribute,
- create_compound_clip,
- swap_clips,
- get_pype_clip_metadata,
- set_project_manager_to_folder_name,
- get_otio_clip_instance_data,
- get_reformated_path
-)
-
-from .api.menu import launch_pype_menu
+from .addon import ResolveAddon
-from .api.plugin import (
- ClipLoader,
- TimelineItemLoader,
- Creator,
- PublishClip
-)
-from .api.workio import (
- open_file,
- save_file,
- current_file,
- has_unsaved_changes,
- file_extensions,
- work_root
+__all__ = (
+ "ResolveAddon",
)
-
-from .api.testing_utils import TestGUI
-
-
-__all__ = [
- # pipeline
- "install",
- "uninstall",
- "ls",
- "containerise",
- "update_container",
- "reload_pipeline",
- "publish",
- "launch_workfiles_app",
- "maintained_selection",
- "remove_instance",
- "list_instances",
-
- # utils
- "setup",
- "get_resolve_module",
-
- # lib
- "maintain_current_timeline",
- "publish_clip_color",
- "get_project_manager",
- "get_current_project",
- "get_current_timeline",
- "create_bin",
- "get_media_pool_item",
- "create_media_pool_item",
- "create_timeline_item",
- "get_timeline_item",
- "get_video_track_names",
- "get_current_timeline_items",
- "get_pype_timeline_item_by_name",
- "get_timeline_item_pype_tag",
- "set_timeline_item_pype_tag",
- "imprint",
- "set_publish_attribute",
- "get_publish_attribute",
- "create_compound_clip",
- "swap_clips",
- "get_pype_clip_metadata",
- "set_project_manager_to_folder_name",
- "get_otio_clip_instance_data",
- "get_reformated_path",
-
- # menu
- "launch_pype_menu",
-
- # plugin
- "ClipLoader",
- "TimelineItemLoader",
- "Creator",
- "PublishClip",
-
- # workio
- "open_file",
- "save_file",
- "current_file",
- "has_unsaved_changes",
- "file_extensions",
- "work_root",
-
- "TestGUI"
-]
diff --git a/openpype/hosts/resolve/addon.py b/openpype/hosts/resolve/addon.py
new file mode 100644
index 00000000000..a31da52a6d5
--- /dev/null
+++ b/openpype/hosts/resolve/addon.py
@@ -0,0 +1,24 @@
+import os
+
+from openpype.modules import OpenPypeModule
+from openpype.modules.interfaces import IHostAddon
+
+from .utils import RESOLVE_ROOT_DIR
+
+
+class ResolveAddon(OpenPypeModule, IHostAddon):
+ name = "resolve"
+ host_name = "resolve"
+
+ def initialize(self, module_settings):
+ self.enabled = True
+
+ def get_launch_hook_paths(self, app):
+ if app.host_name != self.host_name:
+ return []
+ return [
+ os.path.join(RESOLVE_ROOT_DIR, "hooks")
+ ]
+
+ def get_workfile_extensions(self):
+ return [".drp"]
diff --git a/openpype/hosts/resolve/api/__init__.py b/openpype/hosts/resolve/api/__init__.py
index 48bd938e571..cf1edb4c359 100644
--- a/openpype/hosts/resolve/api/__init__.py
+++ b/openpype/hosts/resolve/api/__init__.py
@@ -1,11 +1,137 @@
"""
resolve api
"""
-import os
bmdvr = None
bmdvf = None
-API_DIR = os.path.dirname(os.path.abspath(__file__))
-HOST_DIR = os.path.dirname(API_DIR)
-PLUGINS_DIR = os.path.join(HOST_DIR, "plugins")
+from .utils import (
+ get_resolve_module
+)
+
+from .pipeline import (
+ install,
+ uninstall,
+ ls,
+ containerise,
+ update_container,
+ publish,
+ launch_workfiles_app,
+ maintained_selection,
+ remove_instance,
+ list_instances
+)
+
+from .lib import (
+ maintain_current_timeline,
+ publish_clip_color,
+ get_project_manager,
+ get_current_project,
+ get_current_timeline,
+ create_bin,
+ get_media_pool_item,
+ create_media_pool_item,
+ create_timeline_item,
+ get_timeline_item,
+ get_video_track_names,
+ get_current_timeline_items,
+ get_pype_timeline_item_by_name,
+ get_timeline_item_pype_tag,
+ set_timeline_item_pype_tag,
+ imprint,
+ set_publish_attribute,
+ get_publish_attribute,
+ create_compound_clip,
+ swap_clips,
+ get_pype_clip_metadata,
+ set_project_manager_to_folder_name,
+ get_otio_clip_instance_data,
+ get_reformated_path
+)
+
+from .menu import launch_pype_menu
+
+from .plugin import (
+ ClipLoader,
+ TimelineItemLoader,
+ Creator,
+ PublishClip
+)
+
+from .workio import (
+ open_file,
+ save_file,
+ current_file,
+ has_unsaved_changes,
+ file_extensions,
+ work_root
+)
+
+from .testing_utils import TestGUI
+
+
+__all__ = [
+ "bmdvr",
+ "bmdvf",
+
+ # pipeline
+ "install",
+ "uninstall",
+ "ls",
+ "containerise",
+ "update_container",
+ "reload_pipeline",
+ "publish",
+ "launch_workfiles_app",
+ "maintained_selection",
+ "remove_instance",
+ "list_instances",
+
+ # utils
+ "get_resolve_module",
+
+ # lib
+ "maintain_current_timeline",
+ "publish_clip_color",
+ "get_project_manager",
+ "get_current_project",
+ "get_current_timeline",
+ "create_bin",
+ "get_media_pool_item",
+ "create_media_pool_item",
+ "create_timeline_item",
+ "get_timeline_item",
+ "get_video_track_names",
+ "get_current_timeline_items",
+ "get_pype_timeline_item_by_name",
+ "get_timeline_item_pype_tag",
+ "set_timeline_item_pype_tag",
+ "imprint",
+ "set_publish_attribute",
+ "get_publish_attribute",
+ "create_compound_clip",
+ "swap_clips",
+ "get_pype_clip_metadata",
+ "set_project_manager_to_folder_name",
+ "get_otio_clip_instance_data",
+ "get_reformated_path",
+
+ # menu
+ "launch_pype_menu",
+
+ # plugin
+ "ClipLoader",
+ "TimelineItemLoader",
+ "Creator",
+ "PublishClip",
+
+ # workio
+ "open_file",
+ "save_file",
+ "current_file",
+ "has_unsaved_changes",
+ "file_extensions",
+ "work_root",
+
+ "TestGUI"
+]
diff --git a/openpype/hosts/resolve/api/action.py b/openpype/hosts/resolve/api/action.py
index f8f338a8503..ceedc2cc546 100644
--- a/openpype/hosts/resolve/api/action.py
+++ b/openpype/hosts/resolve/api/action.py
@@ -4,7 +4,7 @@
import pyblish.api
-from ...action import get_errored_instances_from_context
+from openpype.pipeline.publish import get_errored_instances_from_context
class SelectInvalidAction(pyblish.api.Action):
diff --git a/openpype/hosts/resolve/api/lib.py b/openpype/hosts/resolve/api/lib.py
index aa4b2e7219a..f41eb36cafe 100644
--- a/openpype/hosts/resolve/api/lib.py
+++ b/openpype/hosts/resolve/api/lib.py
@@ -4,19 +4,19 @@
import os
import contextlib
from opentimelineio import opentime
-import openpype
-from ..otio import davinci_export as otio_export
+from openpype.lib import Logger
+from openpype.pipeline.editorial import is_overlapping_otio_ranges
-from openpype.api import Logger
+from ..otio import davinci_export as otio_export
-log = Logger().get_logger(__name__)
+log = Logger.get_logger(__name__)
self = sys.modules[__name__]
self.project_manager = None
self.media_storage = None
-# OpenPype sequencial rename variables
+# OpenPype sequential rename variables
self.rename_index = 0
self.rename_add = 0
@@ -59,7 +59,7 @@ def maintain_current_timeline(to_timeline: object,
project = get_current_project()
working_timeline = from_timeline or project.GetCurrentTimeline()
- # swith to the input timeline
+ # switch to the input timeline
project.SetCurrentTimeline(to_timeline)
try:
@@ -319,14 +319,13 @@ def get_current_timeline_items(
selected_track_count = timeline.GetTrackCount(track_type)
# loop all tracks and get items
- _clips = dict()
+ _clips = {}
for track_index in range(1, (int(selected_track_count) + 1)):
_track_name = timeline.GetTrackName(track_type, track_index)
# filter out all unmathed track names
- if track_name:
- if _track_name not in track_name:
- continue
+ if track_name and _track_name not in track_name:
+ continue
timeline_items = timeline.GetItemListInTrack(
track_type, track_index)
@@ -348,12 +347,8 @@ def get_current_timeline_items(
"index": clip_index
}
ti_color = ti.GetClipColor()
- if filter is True:
- if selecting_color in ti_color:
- selected_clips.append(data)
- else:
+ if filter and selecting_color in ti_color or not filter:
selected_clips.append(data)
-
return selected_clips
@@ -566,7 +561,7 @@ def create_compound_clip(clip_data, name, folder):
mp_in_rc = opentime.RationalTime((ci_l_offset), rate)
mp_out_rc = opentime.RationalTime((ci_l_offset + ci_duration - 1), rate)
- # get frame in and out for clip swaping
+ # get frame in and out for clip swapping
in_frame = opentime.to_frames(mp_in_rc)
out_frame = opentime.to_frames(mp_out_rc)
@@ -628,7 +623,7 @@ def create_compound_clip(clip_data, name, folder):
def swap_clips(from_clip, to_clip, to_in_frame, to_out_frame):
"""
- Swaping clips on timeline in timelineItem
+ Swapping clips on timeline in timelineItem
It will add take and activate it to the frame range which is inputted
@@ -699,7 +694,7 @@ def get_pype_clip_metadata(clip):
def get_clip_attributes(clip):
"""
- Collect basic atrributes from resolve timeline item
+ Collect basic attributes from resolve timeline item
Args:
clip (resolve.TimelineItem): timeline item object
@@ -824,7 +819,7 @@ def get_otio_clip_instance_data(otio_timeline, timeline_item_data):
continue
if otio_clip.name not in timeline_item.GetName():
continue
- if openpype.lib.is_overlapping_otio_ranges(
+ if is_overlapping_otio_ranges(
parent_range, timeline_range, strict=True):
# add pypedata marker to otio_clip metadata
diff --git a/openpype/hosts/resolve/api/menu.py b/openpype/hosts/resolve/api/menu.py
index 0d5930d275a..2c7678ee5bb 100644
--- a/openpype/hosts/resolve/api/menu.py
+++ b/openpype/hosts/resolve/api/menu.py
@@ -3,13 +3,13 @@
from Qt import QtWidgets, QtCore
+from openpype.tools.utils import host_tools
+
from .pipeline import (
publish,
launch_workfiles_app
)
-from openpype.tools.utils import host_tools
-
def load_stylesheet():
path = os.path.join(os.path.dirname(__file__), "menu_style.qss")
@@ -54,15 +54,15 @@ def __init__(self, *args, **kwargs):
)
self.setWindowTitle("OpenPype")
- workfiles_btn = QtWidgets.QPushButton("Workfiles ...", self)
- create_btn = QtWidgets.QPushButton("Create ...", self)
- publish_btn = QtWidgets.QPushButton("Publish ...", self)
- load_btn = QtWidgets.QPushButton("Load ...", self)
- inventory_btn = QtWidgets.QPushButton("Inventory ...", self)
- subsetm_btn = QtWidgets.QPushButton("Subset Manager ...", self)
- libload_btn = QtWidgets.QPushButton("Library ...", self)
+ workfiles_btn = QtWidgets.QPushButton("Workfiles...", self)
+ create_btn = QtWidgets.QPushButton("Create...", self)
+ publish_btn = QtWidgets.QPushButton("Publish...", self)
+ load_btn = QtWidgets.QPushButton("Load...", self)
+ inventory_btn = QtWidgets.QPushButton("Inventory...", self)
+ subsetm_btn = QtWidgets.QPushButton("Subset Manager...", self)
+ libload_btn = QtWidgets.QPushButton("Library...", self)
experimental_btn = QtWidgets.QPushButton(
- "Experimental tools ...", self
+ "Experimental tools...", self
)
# rename_btn = QtWidgets.QPushButton("Rename", self)
# set_colorspace_btn = QtWidgets.QPushButton(
diff --git a/openpype/hosts/resolve/api/pipeline.py b/openpype/hosts/resolve/api/pipeline.py
index ce95cfe02ac..1c8d9dc01c7 100644
--- a/openpype/hosts/resolve/api/pipeline.py
+++ b/openpype/hosts/resolve/api/pipeline.py
@@ -4,20 +4,30 @@
import os
import contextlib
from collections import OrderedDict
-from avalon import api as avalon
-from avalon import schema
-from avalon.pipeline import AVALON_CONTAINER_ID
+
from pyblish import api as pyblish
-from openpype.api import Logger
-from . import lib
-from . import PLUGINS_DIR
+
+from openpype.lib import Logger
+from openpype.pipeline import (
+ schema,
+ register_loader_plugin_path,
+ register_creator_plugin_path,
+ deregister_loader_plugin_path,
+ deregister_creator_plugin_path,
+ AVALON_CONTAINER_ID,
+)
from openpype.tools.utils import host_tools
-log = Logger().get_logger(__name__)
+from . import lib
+from .utils import get_resolve_module
+
+log = Logger.get_logger(__name__)
+
+HOST_DIR = os.path.dirname(os.path.abspath(os.path.dirname(__file__)))
+PLUGINS_DIR = os.path.join(HOST_DIR, "plugins")
PUBLISH_PATH = os.path.join(PLUGINS_DIR, "publish")
LOAD_PATH = os.path.join(PLUGINS_DIR, "load")
CREATE_PATH = os.path.join(PLUGINS_DIR, "create")
-INVENTORY_PATH = os.path.join(PLUGINS_DIR, "inventory")
AVALON_CONTAINERS = ":AVALON_CONTAINERS"
@@ -33,19 +43,6 @@ def install():
See the Maya equivalent for inspiration on how to implement this.
"""
- from .. import get_resolve_module
-
- # Disable all families except for the ones we explicitly want to see
- family_states = [
- "imagesequence",
- "render2d",
- "plate",
- "render",
- "mov",
- "clip"
- ]
- avalon.data["familiesStateDefault"] = False
- avalon.data["familiesStateToggled"] = family_states
log.info("openpype.hosts.resolve installed")
@@ -53,9 +50,8 @@ def install():
pyblish.register_plugin_path(PUBLISH_PATH)
log.info("Registering DaVinci Resovle plug-ins..")
- avalon.register_plugin_path(avalon.Loader, LOAD_PATH)
- avalon.register_plugin_path(avalon.Creator, CREATE_PATH)
- avalon.register_plugin_path(avalon.InventoryAction, INVENTORY_PATH)
+ register_loader_plugin_path(LOAD_PATH)
+ register_creator_plugin_path(CREATE_PATH)
# register callback for switching publishable
pyblish.register_callback("instanceToggled", on_pyblish_instance_toggled)
@@ -64,7 +60,7 @@ def install():
def uninstall():
- """Uninstall all tha was installed
+ """Uninstall all that was installed
This is where you undo everything that was done in `install()`.
That means, removing menus, deregistering families and data
@@ -78,9 +74,8 @@ def uninstall():
pyblish.deregister_plugin_path(PUBLISH_PATH)
log.info("Deregistering DaVinci Resovle plug-ins..")
- avalon.deregister_plugin_path(avalon.Loader, LOAD_PATH)
- avalon.deregister_plugin_path(avalon.Creator, CREATE_PATH)
- avalon.deregister_plugin_path(avalon.InventoryAction, INVENTORY_PATH)
+ deregister_loader_plugin_path(LOAD_PATH)
+ deregister_creator_plugin_path(CREATE_PATH)
# register callback for switching publishable
pyblish.deregister_callback("instanceToggled", on_pyblish_instance_toggled)
diff --git a/openpype/hosts/resolve/api/plugin.py b/openpype/hosts/resolve/api/plugin.py
index f1c55a61801..b03125d5026 100644
--- a/openpype/hosts/resolve/api/plugin.py
+++ b/openpype/hosts/resolve/api/plugin.py
@@ -1,13 +1,17 @@
import re
import uuid
-from avalon import api
-import openpype.api as pype
-from openpype.hosts import resolve
-from avalon.vendor import qargparse
-from . import lib
+import qargparse
from Qt import QtWidgets, QtCore
+from openpype.pipeline import (
+ LegacyCreator,
+ LoaderPlugin,
+)
+from openpype.pipeline.context_tools import get_current_project_asset
+from openpype.hosts import resolve
+from . import lib
+
class CreatorWidget(QtWidgets.QDialog):
@@ -133,7 +137,7 @@ def create_row(self, layout, type, text, **kwargs):
# convert label text to normal capitalized text with spaces
label_text = self.camel_case_split(text)
- # assign the new text to lable widget
+ # assign the new text to label widget
label = QtWidgets.QLabel(label_text)
label.setObjectName("LineLabel")
@@ -289,7 +293,7 @@ def __init__(self, cls, context, **options):
""" Initialize object
Arguments:
- cls (avalon.api.Loader): plugin object
+ cls (openpype.pipeline.load.LoaderPlugin): plugin object
context (dict): loader plugin context
options (dict)[optional]: possible keys:
projectBinPath: "path/to/binItem"
@@ -367,11 +371,11 @@ def _populate_data(self):
def _get_asset_data(self):
""" Get all available asset data
- joint `data` key with asset.data dict into the representaion
+ joint `data` key with asset.data dict into the representation
"""
asset_name = self.context["representation"]["context"]["asset"]
- self.data["assetData"] = pype.get_asset(asset_name)["data"]
+ self.data["assetData"] = get_current_project_asset(asset_name)["data"]
def load(self):
# create project bin for the media to be imported into
@@ -445,7 +449,7 @@ def update(self, timeline_item):
return timeline_item
-class TimelineItemLoader(api.Loader):
+class TimelineItemLoader(LoaderPlugin):
"""A basic SequenceLoader for Resolve
This will implement the basic behavior for a loader to inherit from that
@@ -493,7 +497,7 @@ def remove(self, container):
pass
-class Creator(pype.PypeCreatorMixin, api.Creator):
+class Creator(LegacyCreator):
"""Creator class wrapper
"""
marker_color = "Purple"
@@ -502,7 +506,7 @@ def __init__(self, *args, **kwargs):
super(Creator, self).__init__(*args, **kwargs)
from openpype.api import get_current_project_settings
resolve_p_settings = get_current_project_settings().get("resolve")
- self.presets = dict()
+ self.presets = {}
if resolve_p_settings:
self.presets = resolve_p_settings["create"].get(
self.__class__.__name__, {})
@@ -540,8 +544,8 @@ class PublishClip:
"track": "sequence",
}
- # parents search patern
- parents_search_patern = r"\{([a-z]*?)\}"
+ # parents search pattern
+ parents_search_pattern = r"\{([a-z]*?)\}"
# default templates for non-ui use
rename_default = False
@@ -630,7 +634,7 @@ def convert(self):
return self.timeline_item
def _populate_timeline_item_default_data(self):
- """ Populate default formating data from track item. """
+ """ Populate default formatting data from track item. """
self.timeline_item_default_data = {
"_folder_": "shots",
@@ -722,7 +726,7 @@ def _convert_to_tag_data(self):
# mark review layer
if self.review_track and (
self.review_track not in self.review_track_default):
- # if review layer is defined and not the same as defalut
+ # if review layer is defined and not the same as default
self.review_layer = self.review_track
# shot num calculate
if self.rename_index == 0:
@@ -771,7 +775,7 @@ def _convert_to_tag_data(self):
# in case track name and subset name is the same then add
if self.subset_name == self.track_name:
hero_data["subset"] = self.subset
- # assing data to return hierarchy data to tag
+ # assign data to return hierarchy data to tag
tag_hierarchy_data = hero_data
# add data to return data dict
@@ -823,8 +827,8 @@ def _create_parents(self):
""" Create parents and return it in list. """
self.parents = []
- patern = re.compile(self.parents_search_patern)
- par_split = [patern.findall(t).pop()
+ pattern = re.compile(self.parents_search_pattern)
+ par_split = [pattern.findall(t).pop()
for t in self.hierarchy.split("/")]
for key in par_split:
diff --git a/openpype/hosts/resolve/api/preload_console.py b/openpype/hosts/resolve/api/preload_console.py
index 1e3a56b4dde..a822ea24608 100644
--- a/openpype/hosts/resolve/api/preload_console.py
+++ b/openpype/hosts/resolve/api/preload_console.py
@@ -1,9 +1,9 @@
#!/usr/bin/env python
import time
from openpype.hosts.resolve.utils import get_resolve_module
-from openpype.api import Logger
+from openpype.lib import Logger
-log = Logger().get_logger(__name__)
+log = Logger.get_logger(__name__)
wait_delay = 2.5
wait = 0.00
diff --git a/openpype/hosts/resolve/api/testing_utils.py b/openpype/hosts/resolve/api/testing_utils.py
index 98ad6abcf17..4aac66f4b7c 100644
--- a/openpype/hosts/resolve/api/testing_utils.py
+++ b/openpype/hosts/resolve/api/testing_utils.py
@@ -25,12 +25,12 @@ def __init__(self):
ui.Button(
{
"ID": "inputTestSourcesFolder",
- "Text": "Select folder with testing medias",
+ "Text": "Select folder with testing media",
"Weight": 1.25,
"ToolTip": (
"Chose folder with videos, sequences, "
"single images, nested folders with "
- "medias"
+ "media"
),
"Flat": False
}
diff --git a/openpype/hosts/resolve/api/utils.py b/openpype/hosts/resolve/api/utils.py
index 3dee17cb019..871b3af38db 100644
--- a/openpype/hosts/resolve/api/utils.py
+++ b/openpype/hosts/resolve/api/utils.py
@@ -4,21 +4,21 @@
Resolve's tools for setting environment
"""
-import sys
import os
-import shutil
-from . import HOST_DIR
-from openpype.api import Logger
-log = Logger().get_logger(__name__)
+import sys
+
+from openpype.lib import Logger
+
+log = Logger.get_logger(__name__)
def get_resolve_module():
- from openpype.hosts import resolve
+ from openpype.hosts.resolve import api
# dont run if already loaded
- if resolve.api.bmdvr:
+ if api.bmdvr:
log.info(("resolve module is assigned to "
- f"`pype.hosts.resolve.api.bmdvr`: {resolve.api.bmdvr}"))
- return resolve.api.bmdvr
+ f"`pype.hosts.resolve.api.bmdvr`: {api.bmdvr}"))
+ return api.bmdvr
try:
"""
The PYTHONPATH needs to be set correctly for this import
@@ -70,80 +70,10 @@ def get_resolve_module():
sys.exit()
# assign global var and return
bmdvr = bmd.scriptapp("Resolve")
- # bmdvf = bmd.scriptapp("Fusion")
- resolve.api.bmdvr = bmdvr
- resolve.api.bmdvf = bmdvr.Fusion()
+ bmdvf = bmd.scriptapp("Fusion")
+ api.bmdvr = bmdvr
+ api.bmdvf = bmdvf
log.info(("Assigning resolve module to "
- f"`pype.hosts.resolve.api.bmdvr`: {resolve.api.bmdvr}"))
+ f"`pype.hosts.resolve.api.bmdvr`: {api.bmdvr}"))
log.info(("Assigning resolve module to "
- f"`pype.hosts.resolve.api.bmdvf`: {resolve.api.bmdvf}"))
-
-
-def _sync_utility_scripts(env=None):
- """ Synchronizing basic utlility scripts for resolve.
-
- To be able to run scripts from inside `Resolve/Workspace/Scripts` menu
- all scripts has to be accessible from defined folder.
- """
- if not env:
- env = os.environ
-
- # initiate inputs
- scripts = {}
- us_env = env.get("RESOLVE_UTILITY_SCRIPTS_SOURCE_DIR")
- us_dir = env.get("RESOLVE_UTILITY_SCRIPTS_DIR", "")
- us_paths = [os.path.join(
- HOST_DIR,
- "utility_scripts"
- )]
-
- # collect script dirs
- if us_env:
- log.info(f"Utility Scripts Env: `{us_env}`")
- us_paths = us_env.split(
- os.pathsep) + us_paths
-
- # collect scripts from dirs
- for path in us_paths:
- scripts.update({path: os.listdir(path)})
-
- log.info(f"Utility Scripts Dir: `{us_paths}`")
- log.info(f"Utility Scripts: `{scripts}`")
-
- # make sure no script file is in folder
- if next((s for s in os.listdir(us_dir)), None):
- for s in os.listdir(us_dir):
- path = os.path.join(us_dir, s)
- log.info(f"Removing `{path}`...")
- if os.path.isdir(path):
- shutil.rmtree(path, onerror=None)
- else:
- os.remove(path)
-
- # copy scripts into Resolve's utility scripts dir
- for d, sl in scripts.items():
- # directory and scripts list
- for s in sl:
- # script in script list
- src = os.path.join(d, s)
- dst = os.path.join(us_dir, s)
- log.info(f"Copying `{src}` to `{dst}`...")
- if os.path.isdir(src):
- shutil.copytree(
- src, dst, symlinks=False,
- ignore=None, ignore_dangling_symlinks=False
- )
- else:
- shutil.copy2(src, dst)
-
-
-def setup(env=None):
- """ Wrapper installer started from pype.hooks.resolve.ResolvePrelaunch()
- """
- if not env:
- env = os.environ
-
- # synchronize resolve utility scripts
- _sync_utility_scripts(env)
-
- log.info("Resolve OpenPype wrapper has been installed")
+ f"`pype.hosts.resolve.api.bmdvf`: {api.bmdvf}"))
diff --git a/openpype/hosts/resolve/api/workio.py b/openpype/hosts/resolve/api/workio.py
index f1757693877..5a742ecf7e9 100644
--- a/openpype/hosts/resolve/api/workio.py
+++ b/openpype/hosts/resolve/api/workio.py
@@ -2,14 +2,14 @@
import os
from openpype.api import Logger
-from .. import (
+from .lib import (
get_project_manager,
get_current_project,
set_project_manager_to_folder_name
)
-log = Logger().get_logger(__name__)
+log = Logger.get_logger(__name__)
exported_projet_ext = ".drp"
@@ -60,7 +60,7 @@ def open_file(filepath):
# load project from input path
project = pm.LoadProject(fname)
log.info(f"Project {project.GetName()} opened...")
- return True
+
except AttributeError:
log.warning((f"Project with name `{fname}` does not exist! It will "
f"be imported from {filepath} and then loaded..."))
@@ -69,9 +69,8 @@ def open_file(filepath):
project = pm.LoadProject(fname)
log.info(f"Project imported/loaded {project.GetName()}...")
return True
- else:
- return False
-
+ return False
+ return True
def current_file():
pm = get_project_manager()
@@ -80,13 +79,9 @@ def current_file():
name = project.GetName()
fname = name + exported_projet_ext
current_file = os.path.join(current_dir, fname)
- normalised = os.path.normpath(current_file)
-
- # Unsaved current file
- if normalised == "":
+ if not current_file:
return None
-
- return normalised
+ return os.path.normpath(current_file)
def work_root(session):
diff --git a/openpype/hosts/resolve/hooks/pre_resolve_setup.py b/openpype/hosts/resolve/hooks/pre_resolve_setup.py
index bcb27e24fcd..1d977e2d8ea 100644
--- a/openpype/hosts/resolve/hooks/pre_resolve_setup.py
+++ b/openpype/hosts/resolve/hooks/pre_resolve_setup.py
@@ -1,7 +1,7 @@
import os
-import importlib
+
from openpype.lib import PreLaunchHook
-from openpype.hosts.resolve.api import utils
+from openpype.hosts.resolve.utils import setup
class ResolvePrelaunch(PreLaunchHook):
@@ -15,7 +15,7 @@ class ResolvePrelaunch(PreLaunchHook):
def execute(self):
# TODO: add OTIO installation from `openpype/requirements.py`
- # making sure pyton 3.6 is installed at provided path
+ # making sure python 3.6 is installed at provided path
py36_dir = os.path.normpath(
self.launch_context.env.get("PYTHON36_RESOLVE", ""))
assert os.path.isdir(py36_dir), (
@@ -43,18 +43,6 @@ def execute(self):
self.launch_context.env.get("PRE_PYTHON_SCRIPT", ""))
self.launch_context.env["PRE_PYTHON_SCRIPT"] = pre_py_sc
self.log.debug(f"-- pre_py_sc: `{pre_py_sc}`...")
- try:
- __import__("openpype.hosts.resolve")
- __import__("pyblish")
-
- except ImportError:
- self.log.warning(
- "pyblish: Could not load Resolve integration.",
- exc_info=True
- )
- else:
- # Resolve Setup integration
- importlib.reload(utils)
- self.log.debug(f"-- utils.__file__: `{utils.__file__}`")
- utils.setup(self.launch_context.env)
+ # Resolve Setup integration
+ setup(self.launch_context.env)
diff --git a/openpype/hosts/resolve/otio/davinci_export.py b/openpype/hosts/resolve/otio/davinci_export.py
index 2c276d98886..5f11c81fc5d 100644
--- a/openpype/hosts/resolve/otio/davinci_export.py
+++ b/openpype/hosts/resolve/otio/davinci_export.py
@@ -306,7 +306,7 @@ def create_otio_timeline(resolve_project):
if index == 0:
otio_track.append(clip)
else:
- # add previouse otio track to timeline
+ # add previous otio track to timeline
otio_timeline.tracks.append(otio_track)
# convert track to otio
otio_track = create_otio_track(
diff --git a/openpype/hosts/resolve/plugins/create/create_shot_clip.py b/openpype/hosts/resolve/plugins/create/create_shot_clip.py
index 41fdbf5c619..4b14f2493f4 100644
--- a/openpype/hosts/resolve/plugins/create/create_shot_clip.py
+++ b/openpype/hosts/resolve/plugins/create/create_shot_clip.py
@@ -1,9 +1,12 @@
# from pprint import pformat
-from openpype.hosts import resolve
-from openpype.hosts.resolve.api import lib
+from openpype.hosts.resolve.api import plugin, lib
+from openpype.hosts.resolve.api.lib import (
+ get_video_track_names,
+ create_bin,
+)
-class CreateShotClip(resolve.Creator):
+class CreateShotClip(plugin.Creator):
"""Publishable clip"""
label = "Create Publishable Clip"
@@ -11,7 +14,7 @@ class CreateShotClip(resolve.Creator):
icon = "film"
defaults = ["Main"]
- gui_tracks = resolve.get_video_track_names()
+ gui_tracks = get_video_track_names()
gui_name = "OpenPype publish attributes creator"
gui_info = "Define sequential rename and fill hierarchy data."
gui_inputs = {
@@ -116,12 +119,13 @@ class CreateShotClip(resolve.Creator):
"order": 0},
"vSyncTrack": {
"value": gui_tracks, # noqa
- "type": "QComboBox",
- "label": "Hero track",
- "target": "ui",
- "toolTip": "Select driving track name which should be mastering all others", # noqa
- "order": 1}
+ "type": "QComboBox",
+ "label": "Hero track",
+ "target": "ui",
+ "toolTip": "Select driving track name which should be mastering all others", # noqa
+ "order": 1
}
+ }
},
"publishSettings": {
"type": "section",
@@ -135,7 +139,7 @@ class CreateShotClip(resolve.Creator):
"type": "QComboBox",
"label": "Subset Name",
"target": "ui",
- "toolTip": "chose subset name patern, if is selected, name of track layer will be used", # noqa
+ "toolTip": "chose subset name pattern, if is selected, name of track layer will be used", # noqa
"order": 0},
"subsetFamily": {
"value": ["plate", "take"],
@@ -172,28 +176,31 @@ class CreateShotClip(resolve.Creator):
"target": "ui",
"order": 4,
"value": {
- "workfileFrameStart": {
- "value": 1001,
- "type": "QSpinBox",
- "label": "Workfiles Start Frame",
- "target": "tag",
- "toolTip": "Set workfile starting frame number", # noqa
- "order": 0},
- "handleStart": {
- "value": 0,
- "type": "QSpinBox",
- "label": "Handle start (head)",
- "target": "tag",
- "toolTip": "Handle at start of clip", # noqa
- "order": 1},
- "handleEnd": {
- "value": 0,
- "type": "QSpinBox",
- "label": "Handle end (tail)",
- "target": "tag",
- "toolTip": "Handle at end of clip", # noqa
- "order": 2},
- }
+ "workfileFrameStart": {
+ "value": 1001,
+ "type": "QSpinBox",
+ "label": "Workfiles Start Frame",
+ "target": "tag",
+ "toolTip": "Set workfile starting frame number", # noqa
+ "order": 0
+ },
+ "handleStart": {
+ "value": 0,
+ "type": "QSpinBox",
+ "label": "Handle start (head)",
+ "target": "tag",
+ "toolTip": "Handle at start of clip", # noqa
+ "order": 1
+ },
+ "handleEnd": {
+ "value": 0,
+ "type": "QSpinBox",
+ "label": "Handle end (tail)",
+ "target": "tag",
+ "toolTip": "Handle at end of clip", # noqa
+ "order": 2
+ }
+ }
}
}
@@ -229,8 +236,10 @@ def process(self):
v_sync_track = widget.result["vSyncTrack"]["value"]
# sort selected trackItems by
- sorted_selected_track_items = list()
- unsorted_selected_track_items = list()
+ sorted_selected_track_items = []
+ unsorted_selected_track_items = []
+ print("_____ selected ______")
+ print(self.selected)
for track_item_data in self.selected:
if track_item_data["track"]["name"] in v_sync_track:
sorted_selected_track_items.append(track_item_data)
@@ -244,7 +253,7 @@ def process(self):
sq_markers = self.timeline.GetMarkers()
# create media bin for compound clips (trackItems)
- mp_folder = resolve.create_bin(self.timeline.GetName())
+ mp_folder = create_bin(self.timeline.GetName())
kwargs = {
"ui_inputs": widget.result,
@@ -253,11 +262,11 @@ def process(self):
"sq_frame_start": sq_frame_start,
"sq_markers": sq_markers
}
-
+ print(kwargs)
for i, track_item_data in enumerate(sorted_selected_track_items):
self.rename_index = i
-
+ self.log.info(track_item_data)
# convert track item to timeline media pool item
- track_item = resolve.PublishClip(
+ track_item = plugin.PublishClip(
self, track_item_data, **kwargs).convert()
track_item.SetClipColor(lib.publish_clip_color)
diff --git a/openpype/hosts/resolve/plugins/load/load_clip.py b/openpype/hosts/resolve/plugins/load/load_clip.py
index e20384ee6c3..a0c78c182fa 100644
--- a/openpype/hosts/resolve/plugins/load/load_clip.py
+++ b/openpype/hosts/resolve/plugins/load/load_clip.py
@@ -1,12 +1,22 @@
-from avalon import io, api
-from openpype.hosts import resolve
from copy import deepcopy
-from importlib import reload
+
+from openpype.client import (
+ get_version_by_id,
+ get_last_version_by_subset_id,
+)
+# from openpype.hosts import resolve
+from openpype.pipeline import (
+ get_representation_path,
+ legacy_io,
+)
from openpype.hosts.resolve.api import lib, plugin
-reload(plugin)
-reload(lib)
+from openpype.hosts.resolve.api.pipeline import (
+ containerise,
+ update_container,
+)
+
-class LoadClip(resolve.TimelineItemLoader):
+class LoadClip(plugin.TimelineItemLoader):
"""Load a subset to timeline as clip
Place clip to timeline on its asset origin timings collected
@@ -14,7 +24,7 @@ class LoadClip(resolve.TimelineItemLoader):
"""
families = ["render2d", "source", "plate", "render", "review"]
- representations = ["exr", "dpx", "jpg", "jpeg", "png", "h264", ".mov"]
+ representations = ["exr", "dpx", "jpg", "jpeg", "png", "h264", "mov"]
label = "Load as clip"
order = -10
@@ -37,7 +47,7 @@ def load(self, context, name, namespace, options):
})
# load clip to timeline and get main variables
- timeline_item = resolve.ClipLoader(
+ timeline_item = plugin.ClipLoader(
self, context, **options).load()
namespace = namespace or timeline_item.GetName()
version = context['version']
@@ -71,7 +81,7 @@ def load(self, context, name, namespace, options):
self.log.info("Loader done: `{}`".format(name))
- return resolve.containerise(
+ return containerise(
timeline_item,
name, namespace, context,
self.__class__.__name__,
@@ -89,20 +99,18 @@ def update(self, container, representation):
context.update({"representation": representation})
name = container['name']
namespace = container['namespace']
- timeline_item_data = resolve.get_pype_timeline_item_by_name(namespace)
+ timeline_item_data = lib.get_pype_timeline_item_by_name(namespace)
timeline_item = timeline_item_data["clip"]["item"]
- version = io.find_one({
- "type": "version",
- "_id": representation["parent"]
- })
+ project_name = legacy_io.active_project()
+ version = get_version_by_id(project_name, representation["parent"])
version_data = version.get("data", {})
version_name = version.get("name", None)
colorspace = version_data.get("colorspace", None)
object_name = "{}_{}".format(name, namespace)
- self.fname = api.get_representation_path(representation)
+ self.fname = get_representation_path(representation)
context["version"] = {"data": version_data}
- loader = resolve.ClipLoader(self, context)
+ loader = plugin.ClipLoader(self, context)
timeline_item = loader.update(timeline_item)
# add additional metadata from the version to imprint Avalon knob
@@ -129,23 +137,26 @@ def update(self, container, representation):
# update color of clip regarding the version order
self.set_item_color(timeline_item, version)
- return resolve.update_container(timeline_item, data_imprint)
+ return update_container(timeline_item, data_imprint)
@classmethod
def set_item_color(cls, timeline_item, version):
-
# define version name
version_name = version.get("name", None)
# get all versions in list
- versions = io.find({
- "type": "version",
- "parent": version["parent"]
- }).distinct('name')
-
- max_version = max(versions)
+ project_name = legacy_io.active_project()
+ last_version_doc = get_last_version_by_subset_id(
+ project_name,
+ version["parent"],
+ fields=["name"]
+ )
+ if last_version_doc:
+ last_version = last_version_doc["name"]
+ else:
+ last_version = None
# set clip colour
- if version_name == max_version:
+ if version_name == last_version:
timeline_item.SetClipColor(cls.clip_color_last)
else:
timeline_item.SetClipColor(cls.clip_color)
diff --git a/openpype/hosts/resolve/plugins/publish/extract_workfile.py b/openpype/hosts/resolve/plugins/publish/extract_workfile.py
index e3d60465a2b..ea8f19cd8c9 100644
--- a/openpype/hosts/resolve/plugins/publish/extract_workfile.py
+++ b/openpype/hosts/resolve/plugins/publish/extract_workfile.py
@@ -1,7 +1,7 @@
import os
import pyblish.api
import openpype.api
-from openpype.hosts import resolve
+from openpype.hosts.resolve.api.lib import get_project_manager
class ExtractWorkfile(openpype.api.Extractor):
@@ -29,7 +29,7 @@ def process(self, instance):
os.path.join(staging_dir, drp_file_name))
# write out the drp workfile
- resolve.get_project_manager().ExportProject(
+ get_project_manager().ExportProject(
project.GetName(), drp_file_path)
# create drp workfile representation
diff --git a/openpype/hosts/resolve/plugins/publish/precollect_instances.py b/openpype/hosts/resolve/plugins/publish/precollect_instances.py
index 8f1a13a4e5a..8ec169ad656 100644
--- a/openpype/hosts/resolve/plugins/publish/precollect_instances.py
+++ b/openpype/hosts/resolve/plugins/publish/precollect_instances.py
@@ -1,8 +1,14 @@
+from pprint import pformat
+
import pyblish
-from openpype.hosts import resolve
-# # developer reload modules
-from pprint import pformat
+from openpype.hosts.resolve.api.lib import (
+ get_current_timeline_items,
+ get_timeline_item_pype_tag,
+ publish_clip_color,
+ get_publish_attribute,
+ get_otio_clip_instance_data,
+)
class PrecollectInstances(pyblish.api.ContextPlugin):
@@ -14,8 +20,8 @@ class PrecollectInstances(pyblish.api.ContextPlugin):
def process(self, context):
otio_timeline = context.data["otioTimeline"]
- selected_timeline_items = resolve.get_current_timeline_items(
- filter=True, selecting_color=resolve.publish_clip_color)
+ selected_timeline_items = get_current_timeline_items(
+ filter=True, selecting_color=publish_clip_color)
self.log.info(
"Processing enabled track items: {}".format(
@@ -27,7 +33,7 @@ def process(self, context):
timeline_item = timeline_item_data["clip"]["item"]
# get pype tag data
- tag_data = resolve.get_timeline_item_pype_tag(timeline_item)
+ tag_data = get_timeline_item_pype_tag(timeline_item)
self.log.debug(f"__ tag_data: {pformat(tag_data)}")
if not tag_data:
@@ -67,14 +73,15 @@ def process(self, context):
"asset": asset,
"item": timeline_item,
"families": families,
- "publish": resolve.get_publish_attribute(timeline_item),
+ "publish": get_publish_attribute(timeline_item),
"fps": context.data["fps"],
"handleStart": handle_start,
- "handleEnd": handle_end
+ "handleEnd": handle_end,
+ "newAssetPublishing": True
})
# otio clip data
- otio_data = resolve.get_otio_clip_instance_data(
+ otio_data = get_otio_clip_instance_data(
otio_timeline, timeline_item_data) or {}
data.update(otio_data)
@@ -133,7 +140,7 @@ def create_shot_instance(self, context, timeline_item, **data):
"asset": asset,
"family": family,
"families": [],
- "publish": resolve.get_publish_attribute(timeline_item)
+ "publish": get_publish_attribute(timeline_item)
})
context.create_instance(**data)
diff --git a/openpype/hosts/resolve/plugins/publish/precollect_workfile.py b/openpype/hosts/resolve/plugins/publish/precollect_workfile.py
index 1333516177f..0f94216556b 100644
--- a/openpype/hosts/resolve/plugins/publish/precollect_workfile.py
+++ b/openpype/hosts/resolve/plugins/publish/precollect_workfile.py
@@ -1,12 +1,9 @@
import pyblish.api
-from openpype.hosts import resolve
-from avalon import api as avalon
from pprint import pformat
-# dev
-from importlib import reload
+from openpype.hosts.resolve import api as rapi
+from openpype.pipeline import legacy_io
from openpype.hosts.resolve.otio import davinci_export
-reload(davinci_export)
class PrecollectWorkfile(pyblish.api.ContextPlugin):
@@ -17,11 +14,11 @@ class PrecollectWorkfile(pyblish.api.ContextPlugin):
def process(self, context):
- asset = avalon.Session["AVALON_ASSET"]
+ asset = legacy_io.Session["AVALON_ASSET"]
subset = "workfile"
- project = resolve.get_current_project()
+ project = rapi.get_current_project()
fps = project.GetSetting("timelineFrameRate")
- video_tracks = resolve.get_video_track_names()
+ video_tracks = rapi.get_video_track_names()
# adding otio timeline to context
otio_timeline = davinci_export.create_otio_timeline(project)
@@ -31,7 +28,8 @@ def process(self, context):
"asset": asset,
"subset": "{}{}".format(asset, subset.capitalize()),
"item": project,
- "family": "workfile"
+ "family": "workfile",
+ "families": []
}
# create instance with workfile
diff --git a/openpype/hosts/resolve/utility_scripts/OpenPype_sync_util_scripts.py b/openpype/hosts/resolve/utility_scripts/OpenPype_sync_util_scripts.py
index ac66916b915..8f3917bece0 100644
--- a/openpype/hosts/resolve/utility_scripts/OpenPype_sync_util_scripts.py
+++ b/openpype/hosts/resolve/utility_scripts/OpenPype_sync_util_scripts.py
@@ -1,14 +1,16 @@
#!/usr/bin/env python
import os
import sys
-import openpype
+
+from openpype.pipeline import install_host
def main(env):
- import openpype.hosts.resolve as bmdvr
+ from openpype.hosts.resolve.utils import setup
+ import openpype.hosts.resolve.api as bmdvr
# Registers openpype's Global pyblish plugins
- openpype.install()
- bmdvr.setup(env)
+ install_host(bmdvr)
+ setup(env)
if __name__ == "__main__":
diff --git a/openpype/hosts/resolve/utility_scripts/__OpenPype__Menu__.py b/openpype/hosts/resolve/utility_scripts/__OpenPype__Menu__.py
index b1037a9c934..1087a7b7a08 100644
--- a/openpype/hosts/resolve/utility_scripts/__OpenPype__Menu__.py
+++ b/openpype/hosts/resolve/utility_scripts/__OpenPype__Menu__.py
@@ -1,22 +1,17 @@
import os
import sys
-import avalon.api as avalon
-import openpype
-from openpype.api import Logger
+from openpype.pipeline import install_host
+from openpype.lib import Logger
-log = Logger().get_logger(__name__)
+log = Logger.get_logger(__name__)
def main(env):
- import openpype.hosts.resolve as bmdvr
- # Registers openpype's Global pyblish plugins
- openpype.install()
+ import openpype.hosts.resolve.api as bmdvr
# activate resolve from openpype
- avalon.install(bmdvr)
-
- log.info(f"Avalon registred hosts: {avalon.registered_host()}")
+ install_host(bmdvr)
bmdvr.launch_pype_menu()
diff --git a/openpype/hosts/resolve/utility_scripts/tests/test_otio_as_edl.py b/openpype/hosts/resolve/utility_scripts/tests/test_otio_as_edl.py
index 5430ad32dfb..92f2e43a72f 100644
--- a/openpype/hosts/resolve/utility_scripts/tests/test_otio_as_edl.py
+++ b/openpype/hosts/resolve/utility_scripts/tests/test_otio_as_edl.py
@@ -1,11 +1,13 @@
#! python3
import os
import sys
-import avalon.api as avalon
-import openpype
+
import opentimelineio as otio
-from openpype.hosts.resolve import TestGUI
-import openpype.hosts.resolve as bmdvr
+
+from openpype.pipeline import install_host
+
+import openpype.hosts.resolve.api as bmdvr
+from openpype.hosts.resolve.api.testing_utils import TestGUI
from openpype.hosts.resolve.otio import davinci_export as otio_export
@@ -14,10 +16,8 @@ class ThisTestGUI(TestGUI):
def __init__(self):
super(ThisTestGUI, self).__init__()
- # Registers openpype's Global pyblish plugins
- openpype.install()
# activate resolve from openpype
- avalon.install(bmdvr)
+ install_host(bmdvr)
def _open_dir_button_pressed(self, event):
# selected_path = self.fu.RequestFile(os.path.expanduser("~"))
diff --git a/openpype/hosts/resolve/utility_scripts/tests/testing_create_timeline_item_from_path.py b/openpype/hosts/resolve/utility_scripts/tests/testing_create_timeline_item_from_path.py
index afa311e0b8b..91a361ec088 100644
--- a/openpype/hosts/resolve/utility_scripts/tests/testing_create_timeline_item_from_path.py
+++ b/openpype/hosts/resolve/utility_scripts/tests/testing_create_timeline_item_from_path.py
@@ -1,22 +1,25 @@
#! python3
import os
import sys
-import avalon.api as avalon
-import openpype
-from openpype.hosts.resolve import TestGUI
-import openpype.hosts.resolve as bmdvr
+
import clique
+from openpype.pipeline import install_host
+from openpype.hosts.resolve.api.testing_utils import TestGUI
+import openpype.hosts.resolve.api as bmdvr
+from openpype.hosts.resolve.api.lib import (
+ create_media_pool_item,
+ create_timeline_item,
+)
+
class ThisTestGUI(TestGUI):
extensions = [".exr", ".jpg", ".mov", ".png", ".mp4", ".ari", ".arx"]
def __init__(self):
super(ThisTestGUI, self).__init__()
- # Registers openpype's Global pyblish plugins
- openpype.install()
# activate resolve from openpype
- avalon.install(bmdvr)
+ install_host(bmdvr)
def _open_dir_button_pressed(self, event):
# selected_path = self.fu.RequestFile(os.path.expanduser("~"))
@@ -57,10 +60,10 @@ def file_processing(self, fpath):
# skip if unwanted extension
if ext not in self.extensions:
return
- media_pool_item = bmdvr.create_media_pool_item(fpath)
+ media_pool_item = create_media_pool_item(fpath)
print(media_pool_item)
- track_item = bmdvr.create_timeline_item(media_pool_item)
+ track_item = create_timeline_item(media_pool_item)
print(track_item)
diff --git a/openpype/hosts/resolve/utility_scripts/tests/testing_load_media_pool_item.py b/openpype/hosts/resolve/utility_scripts/tests/testing_load_media_pool_item.py
index cfdbe890e59..2e83188bde6 100644
--- a/openpype/hosts/resolve/utility_scripts/tests/testing_load_media_pool_item.py
+++ b/openpype/hosts/resolve/utility_scripts/tests/testing_load_media_pool_item.py
@@ -1,22 +1,24 @@
#! python3
-import avalon.api as avalon
-import openpype
-import openpype.hosts.resolve as bmdvr
+from openpype.pipeline import install_host
+from openpype.hosts.resolve import api as bmdvr
+from openpype.hosts.resolve.api.lib import (
+ create_media_pool_item,
+ create_timeline_item,
+)
def file_processing(fpath):
- media_pool_item = bmdvr.create_media_pool_item(fpath)
+ media_pool_item = create_media_pool_item(fpath)
print(media_pool_item)
- track_item = bmdvr.create_timeline_item(media_pool_item)
+ track_item = create_timeline_item(media_pool_item)
print(track_item)
if __name__ == "__main__":
path = "C:/CODE/__openpype_projects/jtest03dev/shots/sq01/mainsq01sh030/publish/plate/plateMain/v006/jt3d_mainsq01sh030_plateMain_v006.0996.exr"
- openpype.install()
# activate resolve from openpype
- avalon.install(bmdvr)
+ install_host(bmdvr)
- file_processing(path)
\ No newline at end of file
+ file_processing(path)
diff --git a/openpype/hosts/resolve/utils.py b/openpype/hosts/resolve/utils.py
new file mode 100644
index 00000000000..d5c133bbf57
--- /dev/null
+++ b/openpype/hosts/resolve/utils.py
@@ -0,0 +1,54 @@
+import os
+import shutil
+from openpype.lib import Logger
+
+RESOLVE_ROOT_DIR = os.path.dirname(os.path.abspath(__file__))
+
+
+def setup(env):
+ log = Logger.get_logger("ResolveSetup")
+ scripts = {}
+ us_env = env.get("RESOLVE_UTILITY_SCRIPTS_SOURCE_DIR")
+ us_dir = env.get("RESOLVE_UTILITY_SCRIPTS_DIR", "")
+ us_paths = [os.path.join(
+ RESOLVE_ROOT_DIR,
+ "utility_scripts"
+ )]
+
+ # collect script dirs
+ if us_env:
+ log.info("Utility Scripts Env: `{}`".format(us_env))
+ us_paths = us_env.split(
+ os.pathsep) + us_paths
+
+ # collect scripts from dirs
+ for path in us_paths:
+ scripts.update({path: os.listdir(path)})
+
+ log.info("Utility Scripts Dir: `{}`".format(us_paths))
+ log.info("Utility Scripts: `{}`".format(scripts))
+
+ # make sure no script file is in folder
+ for s in os.listdir(us_dir):
+ path = os.path.join(us_dir, s)
+ log.info("Removing `{}`...".format(path))
+ if os.path.isdir(path):
+ shutil.rmtree(path, onerror=None)
+ else:
+ os.remove(path)
+
+ # copy scripts into Resolve's utility scripts dir
+ for d, sl in scripts.items():
+ # directory and scripts list
+ for s in sl:
+ # script in script list
+ src = os.path.join(d, s)
+ dst = os.path.join(us_dir, s)
+ log.info("Copying `{}` to `{}`...".format(src, dst))
+ if os.path.isdir(src):
+ shutil.copytree(
+ src, dst, symlinks=False,
+ ignore=None, ignore_dangling_symlinks=False
+ )
+ else:
+ shutil.copy2(src, dst)
diff --git a/openpype/hosts/standalonepublisher/__init__.py b/openpype/hosts/standalonepublisher/__init__.py
index e69de29bb2d..f47fa6b573d 100644
--- a/openpype/hosts/standalonepublisher/__init__.py
+++ b/openpype/hosts/standalonepublisher/__init__.py
@@ -0,0 +1,6 @@
+from .addon import StandAlonePublishAddon
+
+
+__all__ = (
+ "StandAlonePublishAddon",
+)
diff --git a/openpype/hosts/standalonepublisher/addon.py b/openpype/hosts/standalonepublisher/addon.py
new file mode 100644
index 00000000000..98ec44d4e2b
--- /dev/null
+++ b/openpype/hosts/standalonepublisher/addon.py
@@ -0,0 +1,57 @@
+import os
+
+import click
+
+from openpype.lib import get_openpype_execute_args
+from openpype.lib.execute import run_detached_process
+from openpype.modules import OpenPypeModule
+from openpype.modules.interfaces import ITrayAction, IHostAddon
+
+STANDALONEPUBLISH_ROOT_DIR = os.path.dirname(os.path.abspath(__file__))
+
+
+class StandAlonePublishAddon(OpenPypeModule, ITrayAction, IHostAddon):
+ label = "Publish"
+ name = "standalonepublisher"
+ host_name = "standalonepublisher"
+
+ def initialize(self, modules_settings):
+ self.enabled = modules_settings["standalonepublish_tool"]["enabled"]
+ self.publish_paths = [
+ os.path.join(STANDALONEPUBLISH_ROOT_DIR, "plugins", "publish")
+ ]
+
+ def tray_init(self):
+ return
+
+ def on_action_trigger(self):
+ self.run_standalone_publisher()
+
+ def connect_with_modules(self, enabled_modules):
+ """Collect publish paths from other modules."""
+
+ publish_paths = self.manager.collect_plugin_paths()["publish"]
+ self.publish_paths.extend(publish_paths)
+
+ def run_standalone_publisher(self):
+ args = get_openpype_execute_args("module", self.name, "launch")
+ run_detached_process(args)
+
+ def cli(self, click_group):
+ click_group.add_command(cli_main)
+
+
+@click.group(
+ StandAlonePublishAddon.name,
+ help="StandalonePublisher related commands.")
+def cli_main():
+ pass
+
+
+@cli_main.command()
+def launch():
+ """Launch StandalonePublisher tool UI."""
+
+ from openpype.tools import standalonepublish
+
+ standalonepublish.main()
diff --git a/openpype/hosts/standalonepublisher/plugins/publish/collect_app_name.py b/openpype/hosts/standalonepublisher/plugins/publish/collect_app_name.py
new file mode 100644
index 00000000000..857f3dca205
--- /dev/null
+++ b/openpype/hosts/standalonepublisher/plugins/publish/collect_app_name.py
@@ -0,0 +1,13 @@
+import pyblish.api
+
+
+class CollectSAAppName(pyblish.api.ContextPlugin):
+ """Collect app name and label."""
+
+ label = "Collect App Name/Label"
+ order = pyblish.api.CollectorOrder - 0.5
+ hosts = ["standalonepublisher"]
+
+ def process(self, context):
+ context.data["appName"] = "standalone publisher"
+ context.data["appLabel"] = "Standalone publisher"
diff --git a/openpype/hosts/standalonepublisher/plugins/publish/collect_batch_instances.py b/openpype/hosts/standalonepublisher/plugins/publish/collect_batch_instances.py
deleted file mode 100644
index 4ca1f72cc41..00000000000
--- a/openpype/hosts/standalonepublisher/plugins/publish/collect_batch_instances.py
+++ /dev/null
@@ -1,70 +0,0 @@
-import copy
-import pyblish.api
-from pprint import pformat
-
-
-class CollectBatchInstances(pyblish.api.InstancePlugin):
- """Collect all available instances for batch publish."""
-
- label = "Collect Batch Instances"
- order = pyblish.api.CollectorOrder + 0.489
- hosts = ["standalonepublisher"]
- families = ["background_batch"]
-
- # presets
- default_subset_task = {
- "background_batch": "background"
- }
- subsets = {
- "background_batch": {
- "backgroundLayout": {
- "task": "background",
- "family": "backgroundLayout"
- },
- "backgroundComp": {
- "task": "background",
- "family": "backgroundComp"
- },
- "workfileBackground": {
- "task": "background",
- "family": "workfile"
- }
- }
- }
- unchecked_by_default = []
-
- def process(self, instance):
- context = instance.context
- asset_name = instance.data["asset"]
- family = instance.data["family"]
-
- default_task_name = self.default_subset_task.get(family)
- for subset_name, subset_data in self.subsets[family].items():
- instance_name = f"{asset_name}_{subset_name}"
- task_name = subset_data.get("task") or default_task_name
-
- # create new instance
- new_instance = context.create_instance(instance_name)
-
- # add original instance data except name key
- for key, value in instance.data.items():
- if key not in ["name"]:
- # Make sure value is copy since value may be object which
- # can be shared across all new created objects
- new_instance.data[key] = copy.deepcopy(value)
-
- # add subset data from preset
- new_instance.data.update(subset_data)
-
- new_instance.data["label"] = instance_name
- new_instance.data["subset"] = subset_name
- new_instance.data["task"] = task_name
-
- if subset_name in self.unchecked_by_default:
- new_instance.data["publish"] = False
-
- self.log.info(f"Created new instance: {instance_name}")
- self.log.debug(f"_ inst_data: {pformat(new_instance.data)}")
-
- # delete original instance
- context.remove(instance)
diff --git a/openpype/hosts/standalonepublisher/plugins/publish/collect_bulk_mov_instances.py b/openpype/hosts/standalonepublisher/plugins/publish/collect_bulk_mov_instances.py
index 9f075d66cf2..7925b0ecf3a 100644
--- a/openpype/hosts/standalonepublisher/plugins/publish/collect_bulk_mov_instances.py
+++ b/openpype/hosts/standalonepublisher/plugins/publish/collect_bulk_mov_instances.py
@@ -2,8 +2,8 @@
import json
import pyblish.api
-from avalon import io
-from openpype.lib import get_subset_name_with_asset_doc
+from openpype.client import get_asset_by_name
+from openpype.pipeline.create import get_subset_name
class CollectBulkMovInstances(pyblish.api.InstancePlugin):
@@ -24,12 +24,9 @@ class CollectBulkMovInstances(pyblish.api.InstancePlugin):
def process(self, instance):
context = instance.context
+ project_name = context.data["projectEntity"]["name"]
asset_name = instance.data["asset"]
-
- asset_doc = io.find_one({
- "type": "asset",
- "name": asset_name
- })
+ asset_doc = get_asset_by_name(project_name, asset_name)
if not asset_doc:
raise AssertionError((
"Couldn't find Asset document with name \"{}\""
@@ -47,12 +44,14 @@ def process(self, instance):
task_name = available_task_names[_task_name_low]
break
- subset_name = get_subset_name_with_asset_doc(
+ subset_name = get_subset_name(
self.new_instance_family,
self.subset_name_variant,
task_name,
asset_doc,
- io.Session["AVALON_PROJECT"]
+ project_name,
+ host_name=context.data["hostName"],
+ project_settings=context.data["project_settings"]
)
instance_name = f"{asset_name}_{subset_name}"
diff --git a/openpype/hosts/standalonepublisher/plugins/publish/collect_context.py b/openpype/hosts/standalonepublisher/plugins/publish/collect_context.py
index 6913e0836d0..2bf3917e2f2 100644
--- a/openpype/hosts/standalonepublisher/plugins/publish/collect_context.py
+++ b/openpype/hosts/standalonepublisher/plugins/publish/collect_context.py
@@ -19,7 +19,8 @@
from pprint import pformat
import clique
import pyblish.api
-from avalon import io
+
+from openpype.pipeline import legacy_io
class CollectContextDataSAPublish(pyblish.api.ContextPlugin):
@@ -37,7 +38,7 @@ class CollectContextDataSAPublish(pyblish.api.ContextPlugin):
def process(self, context):
# get json paths from os and load them
- io.install()
+ legacy_io.install()
# get json file context
input_json_path = os.environ.get("SAPUBLISH_INPATH")
@@ -247,7 +248,8 @@ def create_instance(self, context, in_data):
self.log.debug("collecting sequence: {}".format(collections))
instance.data["frameStart"] = int(component["frameStart"])
instance.data["frameEnd"] = int(component["frameEnd"])
- instance.data["fps"] = int(component["fps"])
+ if component.get("fps"):
+ instance.data["fps"] = int(component["fps"])
ext = component["ext"]
if ext.startswith("."):
diff --git a/openpype/hosts/standalonepublisher/plugins/publish/collect_editorial.py b/openpype/hosts/standalonepublisher/plugins/publish/collect_editorial.py
index 0a1d29ccdc0..8633d4bf9dd 100644
--- a/openpype/hosts/standalonepublisher/plugins/publish/collect_editorial.py
+++ b/openpype/hosts/standalonepublisher/plugins/publish/collect_editorial.py
@@ -19,6 +19,7 @@
import opentimelineio as otio
import pyblish.api
from openpype import lib as plib
+from openpype.pipeline.context_tools import get_current_project_asset
class OTIO_View(pyblish.api.Action):
@@ -116,7 +117,7 @@ def process(self, instance):
if extension == ".edl":
# EDL has no frame rate embedded so needs explicit
# frame rate else 24 is asssumed.
- kwargs["rate"] = plib.get_asset()["data"]["fps"]
+ kwargs["rate"] = get_current_project_asset()["data"]["fps"]
instance.data["otio_timeline"] = otio.adapters.read_from_file(
file_path, **kwargs)
diff --git a/openpype/hosts/standalonepublisher/plugins/publish/collect_editorial_instances.py b/openpype/hosts/standalonepublisher/plugins/publish/collect_editorial_instances.py
index 45c6a264ddb..75c260bad70 100644
--- a/openpype/hosts/standalonepublisher/plugins/publish/collect_editorial_instances.py
+++ b/openpype/hosts/standalonepublisher/plugins/publish/collect_editorial_instances.py
@@ -1,8 +1,12 @@
import os
+from copy import deepcopy
+
import opentimelineio as otio
import pyblish.api
+
from openpype import lib as plib
-from copy import deepcopy
+from openpype.pipeline.context_tools import get_current_project_asset
+
class CollectInstances(pyblish.api.InstancePlugin):
"""Collect instances from editorial's OTIO sequence"""
@@ -48,7 +52,7 @@ def process(self, instance):
# get timeline otio data
timeline = instance.data["otio_timeline"]
- fps = plib.get_asset()["data"]["fps"]
+ fps = get_current_project_asset()["data"]["fps"]
tracks = timeline.each_child(
descended_from_type=otio.schema.Track
@@ -83,7 +87,7 @@ def process(self, instance):
if isinstance(clip, otio.schema.Gap):
continue
- # skip all generators like black ampty
+ # skip all generators like black empty
if isinstance(
clip.media_reference,
otio.schema.GeneratorReference):
@@ -142,7 +146,7 @@ def process(self, instance):
"item": clip,
"clipName": clip_name,
- # parent time properities
+ # parent time properties
"trackStartFrame": track_start_frame,
"handleStart": handle_start,
"handleEnd": handle_end,
@@ -166,7 +170,8 @@ def process(self, instance):
"frameStart": frame_start,
"frameEnd": frame_end,
"frameStartH": frame_start - handle_start,
- "frameEndH": frame_end + handle_end
+ "frameEndH": frame_end + handle_end,
+ "newAssetPublishing": True
}
for data_key in instance_data_filter:
@@ -180,14 +185,14 @@ def process(self, instance):
"families": []
}
})
- for subset, properities in self.subsets.items():
- version = properities.get("version")
+ for subset, properties in self.subsets.items():
+ version = properties.get("version")
if version == 0:
- properities.pop("version")
+ properties.pop("version")
# adding Review-able instance
subset_instance_data = deepcopy(instance_data)
- subset_instance_data.update(deepcopy(properities))
+ subset_instance_data.update(deepcopy(properties))
subset_instance_data.update({
# unique attributes
"name": f"{name}_{subset}",
diff --git a/openpype/hosts/standalonepublisher/plugins/publish/collect_editorial_resources.py b/openpype/hosts/standalonepublisher/plugins/publish/collect_editorial_resources.py
index 36bacceb1ca..4d7a13fcf25 100644
--- a/openpype/hosts/standalonepublisher/plugins/publish/collect_editorial_resources.py
+++ b/openpype/hosts/standalonepublisher/plugins/publish/collect_editorial_resources.py
@@ -31,7 +31,7 @@ def process(self, instance):
editorial_source_root = instance.data["editorialSourceRoot"]
editorial_source_path = instance.data["editorialSourcePath"]
- # if `editorial_source_path` then loop trough
+ # if `editorial_source_path` then loop through
if editorial_source_path:
# add family if mov or mp4 found which is longer for
# cutting `trimming` to enable `ExtractTrimmingVideoAudio` plugin
@@ -42,7 +42,7 @@ def process(self, instance):
instance.data["families"] += ["trimming"]
return
- # if template patern in path then fill it with `anatomy_data`
+ # if template pattern in path then fill it with `anatomy_data`
if "{" in editorial_source_root:
editorial_source_root = editorial_source_root.format(
**anatomy_data)
@@ -86,7 +86,7 @@ def process(self, instance):
subset_files.update({clip_dir_path: subset_files_items})
# break the loop if correct_clip_dir was captured
- # no need to cary on if corect folder was found
+ # no need to cary on if correct folder was found
if correct_clip_dir:
break
@@ -113,10 +113,10 @@ def process(self, instance):
unique_subset_names = list()
root_dir = list(subset_files.keys()).pop()
files_list = subset_files[root_dir]
- search_patern = f"({subset}[A-Za-z0-9]+)(?=[\\._\\s])"
+ search_pattern = f"({subset}[A-Za-z0-9]+)(?=[\\._\\s])"
for _file in files_list:
- patern = re.compile(search_patern)
- match = patern.findall(_file)
+ pattern = re.compile(search_pattern)
+ match = pattern.findall(_file)
if not match:
continue
match_subset = match.pop()
@@ -175,7 +175,7 @@ def create_representations(
instance_data["representations"] = list()
collection_head_name = None
- # loop trough collections and create representations
+ # loop through collections and create representations
for _collection in collections:
ext = _collection.tail[1:]
collection_head_name = _collection.head
@@ -210,7 +210,7 @@ def create_representations(
frames.append(frame_start)
frames.append(frame_end)
- # loop trough reminders and create representations
+ # loop through reminders and create representations
for _reminding_file in remainder:
ext = os.path.splitext(_reminding_file)[-1][1:]
if ext not in instance_data["extensions"]:
diff --git a/openpype/hosts/standalonepublisher/plugins/publish/collect_hierarchy.py b/openpype/hosts/standalonepublisher/plugins/publish/collect_hierarchy.py
index acad98d784b..9109bf67266 100644
--- a/openpype/hosts/standalonepublisher/plugins/publish/collect_hierarchy.py
+++ b/openpype/hosts/standalonepublisher/plugins/publish/collect_hierarchy.py
@@ -1,8 +1,11 @@
-import pyblish.api
-import re
import os
-from avalon import io
+from pprint import pformat
+import re
from copy import deepcopy
+import pyblish.api
+
+from openpype.client import get_asset_by_id
+
class CollectHierarchyInstance(pyblish.api.ContextPlugin):
"""Collecting hierarchy context from `parents` and `hierarchy` data
@@ -19,6 +22,7 @@ class CollectHierarchyInstance(pyblish.api.ContextPlugin):
families = ["shot"]
# presets
+ shot_rename = True
shot_rename_template = None
shot_rename_search_patterns = None
shot_add_hierarchy = None
@@ -44,7 +48,7 @@ def rename_with_hierarchy(self, instance):
parent_name = instance.context.data["assetEntity"]["name"]
clip = instance.data["item"]
clip_name = os.path.splitext(clip.name)[0].lower()
- if self.shot_rename_search_patterns:
+ if self.shot_rename_search_patterns and self.shot_rename:
search_text += parent_name + clip_name
instance.data["anatomyData"].update({"clip_name": clip_name})
for type, pattern in self.shot_rename_search_patterns.items():
@@ -54,33 +58,38 @@ def rename_with_hierarchy(self, instance):
continue
instance.data["anatomyData"][type] = match[-1]
- # format to new shot name
- instance.data["asset"] = self.shot_rename_template.format(
- **instance.data["anatomyData"])
+ # format to new shot name
+ instance.data["asset"] = self.shot_rename_template.format(
+ **instance.data["anatomyData"])
def create_hierarchy(self, instance):
- parents = list()
- hierarchy = list()
- visual_hierarchy = [instance.context.data["assetEntity"]]
+ asset_doc = instance.context.data["assetEntity"]
+ project_doc = instance.context.data["projectEntity"]
+ project_name = project_doc["name"]
+ visual_hierarchy = [asset_doc]
+ current_doc = asset_doc
while True:
- visual_parent = io.find_one(
- {"_id": visual_hierarchy[-1]["data"]["visualParent"]}
- )
- if visual_parent:
- visual_hierarchy.append(visual_parent)
- else:
- visual_hierarchy.append(
- instance.context.data["projectEntity"])
+ visual_parent_id = current_doc["data"]["visualParent"]
+ visual_parent = None
+ if visual_parent_id:
+ visual_parent = get_asset_by_id(project_name, visual_parent_id)
+
+ if not visual_parent:
+ visual_hierarchy.append(project_doc)
break
+ visual_hierarchy.append(visual_parent)
+ current_doc = visual_parent
# add current selection context hierarchy from standalonepublisher
+ parents = list()
for entity in reversed(visual_hierarchy):
parents.append({
"entity_type": entity["data"]["entityType"],
"entity_name": entity["name"]
})
- if self.shot_add_hierarchy:
+ hierarchy = list()
+ if self.shot_add_hierarchy.get("enabled"):
parent_template_patern = re.compile(r"\{([a-z]*?)\}")
# fill the parents parts from presets
shot_add_hierarchy = self.shot_add_hierarchy.copy()
@@ -99,7 +108,7 @@ def create_hierarchy(self, instance):
# in case SP context is set to the same folder
if (_index == 0) and ("folder" in parent_key) \
and (parents[-1]["entity_name"] == parent_filled):
- self.log.debug(f" skiping : {parent_filled}")
+ self.log.debug(f" skipping : {parent_filled}")
continue
# in case first parent is project then start parents from start
@@ -119,17 +128,17 @@ def create_hierarchy(self, instance):
# convert hierarchy to string
hierarchy = "/".join(hierarchy)
- # assing to instance data
+ # assign to instance data
instance.data["hierarchy"] = hierarchy
instance.data["parents"] = parents
# print
- self.log.debug(f"Hierarchy: {hierarchy}")
- self.log.debug(f"parents: {parents}")
+ self.log.warning(f"Hierarchy: {hierarchy}")
+ self.log.info(f"parents: {parents}")
+ tasks_to_add = dict()
if self.shot_add_tasks:
- tasks_to_add = dict()
- project_tasks = io.find_one({"type": "project"})["config"]["tasks"]
+ project_tasks = project_doc["config"]["tasks"]
for task_name, task_data in self.shot_add_tasks.items():
_task_data = deepcopy(task_data)
@@ -147,9 +156,7 @@ def create_hierarchy(self, instance):
task_name,
list(project_tasks.keys())))
- instance.data["tasks"] = tasks_to_add
- else:
- instance.data["tasks"] = dict()
+ instance.data["tasks"] = tasks_to_add
# updating hierarchy data
instance.data["anatomyData"].update({
@@ -158,6 +165,9 @@ def create_hierarchy(self, instance):
})
def process(self, context):
+ self.log.info("self.shot_add_hierarchy: {}".format(
+ pformat(self.shot_add_hierarchy)
+ ))
for instance in context:
if instance.data["family"] in self.families:
self.processing_instance(instance)
@@ -202,7 +212,7 @@ def processing_instance(self, instance):
class CollectHierarchyContext(pyblish.api.ContextPlugin):
- '''Collecting Hierarchy from instaces and building
+ '''Collecting Hierarchy from instances and building
context hierarchy tree
'''
diff --git a/openpype/hosts/standalonepublisher/plugins/publish/collect_matching_asset.py b/openpype/hosts/standalonepublisher/plugins/publish/collect_matching_asset.py
index 0d629b1b449..82d7247b2bd 100644
--- a/openpype/hosts/standalonepublisher/plugins/publish/collect_matching_asset.py
+++ b/openpype/hosts/standalonepublisher/plugins/publish/collect_matching_asset.py
@@ -2,9 +2,10 @@
import re
import collections
import pyblish.api
-from avalon import io
from pprint import pformat
+from openpype.client import get_assets
+
class CollectMatchingAssetToInstance(pyblish.api.InstancePlugin):
"""
@@ -118,8 +119,9 @@ def _children_docs(self, documents_by_parent_id, parent_doc):
def _asset_docs_by_parent_id(self, instance):
# Query all assets for project and store them by parent's id to list
+ project_name = instance.context.data["projectEntity"]["name"]
asset_docs_by_parent_id = collections.defaultdict(list)
- for asset_doc in io.find({"type": "asset"}):
+ for asset_doc in get_assets(project_name):
parent_id = asset_doc["data"]["visualParent"]
asset_docs_by_parent_id[parent_id].append(asset_doc)
return asset_docs_by_parent_id
diff --git a/openpype/hosts/standalonepublisher/plugins/publish/collect_original_basename.py b/openpype/hosts/standalonepublisher/plugins/publish/collect_original_basename.py
new file mode 100644
index 00000000000..b83a924d33a
--- /dev/null
+++ b/openpype/hosts/standalonepublisher/plugins/publish/collect_original_basename.py
@@ -0,0 +1,18 @@
+# -*- coding: utf-8 -*-
+"""Collect original base name for use in templates."""
+from pathlib import Path
+
+import pyblish.api
+
+
+class CollectOriginalBasename(pyblish.api.InstancePlugin):
+ """Collect original file base name."""
+
+ order = pyblish.api.CollectorOrder + 0.498
+ label = "Collect Base Name"
+ hosts = ["standalonepublisher"]
+ families = ["simpleUnrealTexture"]
+
+ def process(self, instance):
+ file_name = Path(instance.data["representations"][0]["files"])
+ instance.data["originalBasename"] = file_name.stem
diff --git a/openpype/hosts/standalonepublisher/plugins/publish/collect_representation_names.py b/openpype/hosts/standalonepublisher/plugins/publish/collect_representation_names.py
index c9063c22edb..82dbba3345c 100644
--- a/openpype/hosts/standalonepublisher/plugins/publish/collect_representation_names.py
+++ b/openpype/hosts/standalonepublisher/plugins/publish/collect_representation_names.py
@@ -8,7 +8,7 @@ class CollectRepresentationNames(pyblish.api.InstancePlugin):
Sets the representation names for given families based on RegEx filter
"""
- label = "Collect Representaion Names"
+ label = "Collect Representation Names"
order = pyblish.api.CollectorOrder
families = []
hosts = ["standalonepublisher"]
diff --git a/openpype/hosts/standalonepublisher/plugins/publish/collect_texture.py b/openpype/hosts/standalonepublisher/plugins/publish/collect_texture.py
index 596a8ccfd25..c1c48ec72d9 100644
--- a/openpype/hosts/standalonepublisher/plugins/publish/collect_texture.py
+++ b/openpype/hosts/standalonepublisher/plugins/publish/collect_texture.py
@@ -3,9 +3,10 @@
import pyblish.api
import json
-from avalon.api import format_template_with_optional_keys
-
-from openpype.lib import prepare_template_data
+from openpype.lib import (
+ prepare_template_data,
+ StringTemplate,
+)
class CollectTextures(pyblish.api.ContextPlugin):
@@ -81,14 +82,10 @@ def process(self, context):
parsed_subset = instance.data["subset"].replace(
instance.data["family"], '')
- fill_pairs = {
+ explicit_data = {
"subset": parsed_subset
}
- fill_pairs = prepare_template_data(fill_pairs)
- workfile_subset = format_template_with_optional_keys(
- fill_pairs, self.workfile_subset_template)
-
processed_instance = False
for repre in instance.data["representations"]:
ext = repre["ext"].replace('.', '')
@@ -102,6 +99,22 @@ def process(self, context):
if ext in self.main_workfile_extensions or \
ext in self.other_workfile_extensions:
+ formatting_data = self._get_parsed_groups(
+ repre_file,
+ self.input_naming_patterns["workfile"],
+ self.input_naming_groups["workfile"],
+ self.color_space
+ )
+ self.log.info("Parsed groups from workfile "
+ "name '{}': {}".format(repre_file,
+ formatting_data))
+
+ formatting_data.update(explicit_data)
+ fill_pairs = prepare_template_data(formatting_data)
+ workfile_subset = StringTemplate.format_strict_template(
+ self.workfile_subset_template, fill_pairs
+ )
+
asset_build = self._get_asset_build(
repre_file,
self.input_naming_patterns["workfile"],
@@ -148,11 +161,23 @@ def process(self, context):
resource_files[workfile_subset].append(item)
if ext in self.texture_extensions:
+ formatting_data = self._get_parsed_groups(
+ repre_file,
+ self.input_naming_patterns["textures"],
+ self.input_naming_groups["textures"],
+ self.color_space
+ )
+
+ self.log.info("Parsed groups from texture "
+ "name '{}': {}".format(repre_file,
+ formatting_data))
+
c_space = self._get_color_space(
repre_file,
self.color_space
)
+ # optional value
channel = self._get_channel_name(
repre_file,
self.input_naming_patterns["textures"],
@@ -160,6 +185,7 @@ def process(self, context):
self.color_space
)
+ # optional value
shader = self._get_shader_name(
repre_file,
self.input_naming_patterns["textures"],
@@ -167,16 +193,19 @@ def process(self, context):
self.color_space
)
- formatting_data = {
+ explicit_data = {
"color_space": c_space or '', # None throws exception
"channel": channel or '',
"shader": shader or '',
"subset": parsed_subset or ''
}
+ formatting_data.update(explicit_data)
+
fill_pairs = prepare_template_data(formatting_data)
- subset = format_template_with_optional_keys(
- fill_pairs, self.texture_subset_template)
+ subset = StringTemplate.format_strict_template(
+ self.texture_subset_template, fill_pairs
+ )
asset_build = self._get_asset_build(
repre_file,
@@ -243,6 +272,13 @@ def _create_new_instances(self, context, asset, asset_builds,
for asset_build, version, subset, family in asset_builds:
if not main_version:
main_version = version
+
+ try:
+ version_int = int(version or main_version or 1)
+ except ValueError:
+ self.log.error("Parsed version {} is not "
+ "an number".format(version))
+
new_instance = context.create_instance(subset)
new_instance.data.update(
{
@@ -251,7 +287,7 @@ def _create_new_instances(self, context, asset, asset_builds,
"label": subset,
"name": subset,
"family": family,
- "version": int(version or main_version or 1),
+ "version": version_int,
"asset_build": asset_build # remove in validator
}
)
@@ -320,13 +356,14 @@ def _get_asset_build(self, name,
"""
asset_name = "NOT_AVAIL"
- return self._parse(name, input_naming_patterns, input_naming_groups,
- color_spaces, 'asset') or asset_name
+ return (self._parse_key(name, input_naming_patterns,
+ input_naming_groups, color_spaces, 'asset') or
+ asset_name)
def _get_version(self, name, input_naming_patterns, input_naming_groups,
color_spaces):
- found = self._parse(name, input_naming_patterns, input_naming_groups,
- color_spaces, 'version')
+ found = self._parse_key(name, input_naming_patterns,
+ input_naming_groups, color_spaces, 'version')
if found:
return found.replace('v', '')
@@ -336,8 +373,8 @@ def _get_version(self, name, input_naming_patterns, input_naming_groups,
def _get_udim(self, name, input_naming_patterns, input_naming_groups,
color_spaces):
"""Parses from 'name' udim value."""
- found = self._parse(name, input_naming_patterns, input_naming_groups,
- color_spaces, 'udim')
+ found = self._parse_key(name, input_naming_patterns,
+ input_naming_groups, color_spaces, 'udim')
if found:
return found
@@ -375,12 +412,15 @@ def _get_shader_name(self, name, input_naming_patterns,
Unknown format of channel name and color spaces >> cs are known
list - 'color_space' used as a placeholder
"""
- found = self._parse(name, input_naming_patterns, input_naming_groups,
- color_spaces, 'shader')
- if found:
- return found
+ found = None
+ try:
+ found = self._parse_key(name, input_naming_patterns,
+ input_naming_groups, color_spaces,
+ 'shader')
+ except ValueError:
+ self.log.warning("Didn't find shader in {}".format(name))
- self.log.warning("Didn't find shader in {}".format(name))
+ return found
def _get_channel_name(self, name, input_naming_patterns,
input_naming_groups, color_spaces):
@@ -389,15 +429,18 @@ def _get_channel_name(self, name, input_naming_patterns,
Unknown format of channel name and color spaces >> cs are known
list - 'color_space' used as a placeholder
"""
- found = self._parse(name, input_naming_patterns, input_naming_groups,
- color_spaces, 'channel')
- if found:
- return found
-
- self.log.warning("Didn't find channel in {}".format(name))
-
- def _parse(self, name, input_naming_patterns, input_naming_groups,
- color_spaces, key):
+ found = None
+ try:
+ found = self._parse_key(name, input_naming_patterns,
+ input_naming_groups, color_spaces,
+ 'channel')
+ except ValueError:
+ self.log.warning("Didn't find channel in {}".format(name))
+
+ return found
+
+ def _parse_key(self, name, input_naming_patterns, input_naming_groups,
+ color_spaces, key):
"""Universal way to parse 'name' with configurable regex groups.
Args:
@@ -411,23 +454,47 @@ def _parse(self, name, input_naming_patterns, input_naming_groups,
Raises:
ValueError - if broken 'input_naming_groups'
"""
+ parsed_groups = self._get_parsed_groups(name,
+ input_naming_patterns,
+ input_naming_groups,
+ color_spaces)
+
+ try:
+ parsed_value = parsed_groups[key]
+ return parsed_value
+ except (IndexError, KeyError):
+ msg = ("'Textures group positions' must " +
+ "have '{}' key".format(key))
+ raise ValueError(msg)
+
+ def _get_parsed_groups(self, name, input_naming_patterns,
+ input_naming_groups, color_spaces):
+ """Universal way to parse 'name' with configurable regex groups.
+
+ Args:
+ name (str): workfile name or texture name
+ input_naming_patterns (list):
+ [workfile_pattern] or [texture_pattern]
+ input_naming_groups (list)
+ ordinal position of regex groups matching to input_naming..
+ color_spaces (list) - predefined color spaces
+
+ Returns:
+ (dict) {group_name:parsed_value}
+ """
for input_pattern in input_naming_patterns:
for cs in color_spaces:
pattern = input_pattern.replace('{color_space}', cs)
regex_result = re.findall(pattern, name)
if regex_result:
- idx = list(input_naming_groups).index(key)
- if idx < 0:
- msg = "input_naming_groups must " +\
- "have '{}' key".format(key)
- raise ValueError(msg)
-
- try:
- parsed_value = regex_result[0][idx]
- return parsed_value
- except IndexError:
- self.log.warning("Wrong index, probably "
- "wrong name {}".format(name))
+ if len(regex_result[0]) == len(input_naming_groups):
+ return dict(zip(input_naming_groups, regex_result[0]))
+ else:
+ self.log.warning("No of parsed groups doesn't match "
+ "no of group labels")
+
+ raise ValueError("Name '{}' cannot be parsed by any "
+ "'{}' patterns".format(name, input_naming_patterns))
def _update_representations(self, upd_representations):
"""Frames dont have sense for textures, add collected udims instead."""
diff --git a/openpype/hosts/standalonepublisher/plugins/publish/extract_bg_for_compositing.py b/openpype/hosts/standalonepublisher/plugins/publish/extract_bg_for_compositing.py
deleted file mode 100644
index f07499c15d4..00000000000
--- a/openpype/hosts/standalonepublisher/plugins/publish/extract_bg_for_compositing.py
+++ /dev/null
@@ -1,242 +0,0 @@
-import os
-import json
-import copy
-import openpype.api
-from avalon import io
-
-PSDImage = None
-
-
-class ExtractBGForComp(openpype.api.Extractor):
- label = "Extract Background for Compositing"
- families = ["backgroundComp"]
- hosts = ["standalonepublisher"]
-
- new_instance_family = "background"
-
- # Presetable
- allowed_group_names = [
- "OL", "BG", "MG", "FG", "SB", "UL", "SKY", "Field Guide", "Field_Guide",
- "ANIM"
- ]
-
- def process(self, instance):
- # Check if python module `psd_tools` is installed
- try:
- global PSDImage
- from psd_tools import PSDImage
- except Exception:
- raise AssertionError(
- "BUG: Python module `psd-tools` is not installed!"
- )
-
- self.allowed_group_names = [
- name.lower()
- for name in self.allowed_group_names
- ]
-
- self.redo_global_plugins(instance)
-
- repres = instance.data.get("representations")
- if not repres:
- self.log.info("There are no representations on instance.")
- return
-
- if not instance.data.get("transfers"):
- instance.data["transfers"] = []
-
- # Prepare staging dir
- staging_dir = self.staging_dir(instance)
- if not os.path.exists(staging_dir):
- os.makedirs(staging_dir)
-
- for repre in tuple(repres):
- # Skip all files without .psd extension
- repre_ext = repre["ext"].lower()
- if repre_ext.startswith("."):
- repre_ext = repre_ext[1:]
-
- if repre_ext != "psd":
- continue
-
- # Prepare publish dir for transfers
- publish_dir = instance.data["publishDir"]
-
- # Prepare json filepath where extracted metadata are stored
- json_filename = "{}.json".format(instance.name)
- json_full_path = os.path.join(staging_dir, json_filename)
-
- self.log.debug(f"`staging_dir` is \"{staging_dir}\"")
-
- # Prepare new repre data
- new_repre = {
- "name": "json",
- "ext": "json",
- "files": json_filename,
- "stagingDir": staging_dir
- }
-
- # TODO add check of list
- psd_filename = repre["files"]
- psd_folder_path = repre["stagingDir"]
- psd_filepath = os.path.join(psd_folder_path, psd_filename)
- self.log.debug(f"psd_filepath: \"{psd_filepath}\"")
- psd_object = PSDImage.open(psd_filepath)
-
- json_data, transfers = self.export_compositing_images(
- psd_object, staging_dir, publish_dir
- )
- self.log.info("Json file path: {}".format(json_full_path))
- with open(json_full_path, "w") as json_filestream:
- json.dump(json_data, json_filestream, indent=4)
-
- instance.data["transfers"].extend(transfers)
- instance.data["representations"].remove(repre)
- instance.data["representations"].append(new_repre)
-
- def export_compositing_images(self, psd_object, output_dir, publish_dir):
- json_data = {
- "__schema_version__": 1,
- "children": []
- }
- transfers = []
- for main_idx, main_layer in enumerate(psd_object):
- if (
- not main_layer.is_visible()
- or main_layer.name.lower() not in self.allowed_group_names
- or not main_layer.is_group
- ):
- continue
-
- export_layers = []
- layers_idx = 0
- for layer in main_layer:
- # TODO this way may be added also layers next to "ADJ"
- if layer.name.lower() == "adj":
- for _layer in layer:
- export_layers.append((layers_idx, _layer))
- layers_idx += 1
-
- else:
- export_layers.append((layers_idx, layer))
- layers_idx += 1
-
- if not export_layers:
- continue
-
- main_layer_data = {
- "index": main_idx,
- "name": main_layer.name,
- "children": []
- }
-
- for layer_idx, layer in export_layers:
- has_size = layer.width > 0 and layer.height > 0
- if not has_size:
- self.log.debug((
- "Skipping layer \"{}\" because does "
- "not have any content."
- ).format(layer.name))
- continue
-
- main_layer_name = main_layer.name.replace(" ", "_")
- layer_name = layer.name.replace(" ", "_")
-
- filename = "{:0>2}_{}_{:0>2}_{}.png".format(
- main_idx + 1, main_layer_name, layer_idx + 1, layer_name
- )
- layer_data = {
- "index": layer_idx,
- "name": layer.name,
- "filename": filename
- }
- output_filepath = os.path.join(output_dir, filename)
- dst_filepath = os.path.join(publish_dir, filename)
- transfers.append((output_filepath, dst_filepath))
-
- pil_object = layer.composite(viewport=psd_object.viewbox)
- pil_object.save(output_filepath, "PNG")
-
- main_layer_data["children"].append(layer_data)
-
- if main_layer_data["children"]:
- json_data["children"].append(main_layer_data)
-
- return json_data, transfers
-
- def redo_global_plugins(self, instance):
- # TODO do this in collection phase
- # Copy `families` and check if `family` is not in current families
- families = instance.data.get("families") or list()
- if families:
- families = list(set(families))
-
- if self.new_instance_family in families:
- families.remove(self.new_instance_family)
-
- self.log.debug(
- "Setting new instance families {}".format(str(families))
- )
- instance.data["families"] = families
-
- # Override instance data with new information
- instance.data["family"] = self.new_instance_family
-
- subset_name = instance.data["anatomyData"]["subset"]
- asset_doc = instance.data["assetEntity"]
- latest_version = self.find_last_version(subset_name, asset_doc)
- version_number = 1
- if latest_version is not None:
- version_number += latest_version
-
- instance.data["latestVersion"] = latest_version
- instance.data["version"] = version_number
-
- # Same data apply to anatomy data
- instance.data["anatomyData"].update({
- "family": self.new_instance_family,
- "version": version_number
- })
-
- # Redo publish and resources dir
- anatomy = instance.context.data["anatomy"]
- template_data = copy.deepcopy(instance.data["anatomyData"])
- template_data.update({
- "frame": "FRAME_TEMP",
- "representation": "TEMP"
- })
- anatomy_filled = anatomy.format(template_data)
- if "folder" in anatomy.templates["publish"]:
- publish_folder = anatomy_filled["publish"]["folder"]
- else:
- publish_folder = os.path.dirname(anatomy_filled["publish"]["path"])
-
- publish_folder = os.path.normpath(publish_folder)
- resources_folder = os.path.join(publish_folder, "resources")
-
- instance.data["publishDir"] = publish_folder
- instance.data["resourcesDir"] = resources_folder
-
- self.log.debug("publishDir: \"{}\"".format(publish_folder))
- self.log.debug("resourcesDir: \"{}\"".format(resources_folder))
-
- def find_last_version(self, subset_name, asset_doc):
- subset_doc = io.find_one({
- "type": "subset",
- "name": subset_name,
- "parent": asset_doc["_id"]
- })
-
- if subset_doc is None:
- self.log.debug("Subset entity does not exist yet.")
- else:
- version_doc = io.find_one(
- {
- "type": "version",
- "parent": subset_doc["_id"]
- },
- sort=[("name", -1)]
- )
- if version_doc:
- return int(version_doc["name"])
- return None
diff --git a/openpype/hosts/standalonepublisher/plugins/publish/extract_bg_main_groups.py b/openpype/hosts/standalonepublisher/plugins/publish/extract_bg_main_groups.py
deleted file mode 100644
index 2c92366ae9f..00000000000
--- a/openpype/hosts/standalonepublisher/plugins/publish/extract_bg_main_groups.py
+++ /dev/null
@@ -1,246 +0,0 @@
-import os
-import copy
-import json
-import openpype.api
-import pyblish.api
-from avalon import io
-
-PSDImage = None
-
-
-class ExtractBGMainGroups(openpype.api.Extractor):
- label = "Extract Background Layout"
- order = pyblish.api.ExtractorOrder + 0.02
- families = ["backgroundLayout"]
- hosts = ["standalonepublisher"]
-
- new_instance_family = "background"
-
- # Presetable
- allowed_group_names = [
- "OL", "BG", "MG", "FG", "UL", "SB", "SKY", "Field Guide", "Field_Guide",
- "ANIM"
- ]
-
- def process(self, instance):
- # Check if python module `psd_tools` is installed
- try:
- global PSDImage
- from psd_tools import PSDImage
- except Exception:
- raise AssertionError(
- "BUG: Python module `psd-tools` is not installed!"
- )
-
- self.allowed_group_names = [
- name.lower()
- for name in self.allowed_group_names
- ]
- repres = instance.data.get("representations")
- if not repres:
- self.log.info("There are no representations on instance.")
- return
-
- self.redo_global_plugins(instance)
-
- repres = instance.data.get("representations")
- if not repres:
- self.log.info("There are no representations on instance.")
- return
-
- if not instance.data.get("transfers"):
- instance.data["transfers"] = []
-
- # Prepare staging dir
- staging_dir = self.staging_dir(instance)
- if not os.path.exists(staging_dir):
- os.makedirs(staging_dir)
-
- # Prepare publish dir for transfers
- publish_dir = instance.data["publishDir"]
-
- for repre in tuple(repres):
- # Skip all files without .psd extension
- repre_ext = repre["ext"].lower()
- if repre_ext.startswith("."):
- repre_ext = repre_ext[1:]
-
- if repre_ext != "psd":
- continue
-
- # Prepare json filepath where extracted metadata are stored
- json_filename = "{}.json".format(instance.name)
- json_full_path = os.path.join(staging_dir, json_filename)
-
- self.log.debug(f"`staging_dir` is \"{staging_dir}\"")
-
- # Prepare new repre data
- new_repre = {
- "name": "json",
- "ext": "json",
- "files": json_filename,
- "stagingDir": staging_dir
- }
-
- # TODO add check of list
- psd_filename = repre["files"]
- psd_folder_path = repre["stagingDir"]
- psd_filepath = os.path.join(psd_folder_path, psd_filename)
- self.log.debug(f"psd_filepath: \"{psd_filepath}\"")
- psd_object = PSDImage.open(psd_filepath)
-
- json_data, transfers = self.export_compositing_images(
- psd_object, staging_dir, publish_dir
- )
- self.log.info("Json file path: {}".format(json_full_path))
- with open(json_full_path, "w") as json_filestream:
- json.dump(json_data, json_filestream, indent=4)
-
- instance.data["transfers"].extend(transfers)
- instance.data["representations"].remove(repre)
- instance.data["representations"].append(new_repre)
-
- def export_compositing_images(self, psd_object, output_dir, publish_dir):
- json_data = {
- "__schema_version__": 1,
- "children": []
- }
- output_ext = ".png"
-
- to_export = []
- for layer_idx, layer in enumerate(psd_object):
- layer_name = layer.name.replace(" ", "_")
- if (
- not layer.is_visible()
- or layer_name.lower() not in self.allowed_group_names
- ):
- continue
-
- has_size = layer.width > 0 and layer.height > 0
- if not has_size:
- self.log.debug((
- "Skipping layer \"{}\" because does not have any content."
- ).format(layer.name))
- continue
-
- filebase = "{:0>2}_{}".format(layer_idx, layer_name)
- if layer_name.lower() == "anim":
- if not layer.is_group:
- self.log.warning("ANIM layer is not a group layer.")
- continue
-
- children = []
- for anim_idx, anim_layer in enumerate(layer):
- anim_layer_name = anim_layer.name.replace(" ", "_")
- filename = "{}_{:0>2}_{}{}".format(
- filebase, anim_idx, anim_layer_name, output_ext
- )
- children.append({
- "index": anim_idx,
- "name": anim_layer.name,
- "filename": filename
- })
- to_export.append((anim_layer, filename))
-
- json_data["children"].append({
- "index": layer_idx,
- "name": layer.name,
- "children": children
- })
- continue
-
- filename = filebase + output_ext
- json_data["children"].append({
- "index": layer_idx,
- "name": layer.name,
- "filename": filename
- })
- to_export.append((layer, filename))
-
- transfers = []
- for layer, filename in to_export:
- output_filepath = os.path.join(output_dir, filename)
- dst_filepath = os.path.join(publish_dir, filename)
- transfers.append((output_filepath, dst_filepath))
-
- pil_object = layer.composite(viewport=psd_object.viewbox)
- pil_object.save(output_filepath, "PNG")
-
- return json_data, transfers
-
- def redo_global_plugins(self, instance):
- # TODO do this in collection phase
- # Copy `families` and check if `family` is not in current families
- families = instance.data.get("families") or list()
- if families:
- families = list(set(families))
-
- if self.new_instance_family in families:
- families.remove(self.new_instance_family)
-
- self.log.debug(
- "Setting new instance families {}".format(str(families))
- )
- instance.data["families"] = families
-
- # Override instance data with new information
- instance.data["family"] = self.new_instance_family
-
- subset_name = instance.data["anatomyData"]["subset"]
- asset_doc = instance.data["assetEntity"]
- latest_version = self.find_last_version(subset_name, asset_doc)
- version_number = 1
- if latest_version is not None:
- version_number += latest_version
-
- instance.data["latestVersion"] = latest_version
- instance.data["version"] = version_number
-
- # Same data apply to anatomy data
- instance.data["anatomyData"].update({
- "family": self.new_instance_family,
- "version": version_number
- })
-
- # Redo publish and resources dir
- anatomy = instance.context.data["anatomy"]
- template_data = copy.deepcopy(instance.data["anatomyData"])
- template_data.update({
- "frame": "FRAME_TEMP",
- "representation": "TEMP"
- })
- anatomy_filled = anatomy.format(template_data)
- if "folder" in anatomy.templates["publish"]:
- publish_folder = anatomy_filled["publish"]["folder"]
- else:
- publish_folder = os.path.dirname(anatomy_filled["publish"]["path"])
-
- publish_folder = os.path.normpath(publish_folder)
- resources_folder = os.path.join(publish_folder, "resources")
-
- instance.data["publishDir"] = publish_folder
- instance.data["resourcesDir"] = resources_folder
-
- self.log.debug("publishDir: \"{}\"".format(publish_folder))
- self.log.debug("resourcesDir: \"{}\"".format(resources_folder))
-
- def find_last_version(self, subset_name, asset_doc):
- subset_doc = io.find_one({
- "type": "subset",
- "name": subset_name,
- "parent": asset_doc["_id"]
- })
-
- if subset_doc is None:
- self.log.debug("Subset entity does not exist yet.")
- else:
- version_doc = io.find_one(
- {
- "type": "version",
- "parent": subset_doc["_id"]
- },
- sort=[("name", -1)]
- )
- if version_doc:
- return int(version_doc["name"])
- return None
diff --git a/openpype/hosts/standalonepublisher/plugins/publish/extract_images_from_psd.py b/openpype/hosts/standalonepublisher/plugins/publish/extract_images_from_psd.py
deleted file mode 100644
index e3094b2e3fa..00000000000
--- a/openpype/hosts/standalonepublisher/plugins/publish/extract_images_from_psd.py
+++ /dev/null
@@ -1,170 +0,0 @@
-import os
-import copy
-import openpype.api
-import pyblish.api
-from avalon import io
-
-PSDImage = None
-
-
-class ExtractImagesFromPSD(openpype.api.Extractor):
- # PLUGIN is not currently enabled because was decided to use different
- # approach
- enabled = False
- active = False
- label = "Extract Images from PSD"
- order = pyblish.api.ExtractorOrder + 0.02
- families = ["backgroundLayout"]
- hosts = ["standalonepublisher"]
-
- new_instance_family = "image"
- ignored_instance_data_keys = ("name", "label", "stagingDir", "version")
- # Presetable
- allowed_group_names = [
- "OL", "BG", "MG", "FG", "UL", "SKY", "Field Guide", "Field_Guide",
- "ANIM"
- ]
-
- def process(self, instance):
- # Check if python module `psd_tools` is installed
- try:
- global PSDImage
- from psd_tools import PSDImage
- except Exception:
- raise AssertionError(
- "BUG: Python module `psd-tools` is not installed!"
- )
-
- self.allowed_group_names = [
- name.lower()
- for name in self.allowed_group_names
- ]
- repres = instance.data.get("representations")
- if not repres:
- self.log.info("There are no representations on instance.")
- return
-
- for repre in tuple(repres):
- # Skip all files without .psd extension
- repre_ext = repre["ext"].lower()
- if repre_ext.startswith("."):
- repre_ext = repre_ext[1:]
-
- if repre_ext != "psd":
- continue
-
- # TODO add check of list of "files" value
- psd_filename = repre["files"]
- psd_folder_path = repre["stagingDir"]
- psd_filepath = os.path.join(psd_folder_path, psd_filename)
- self.log.debug(f"psd_filepath: \"{psd_filepath}\"")
- psd_object = PSDImage.open(psd_filepath)
-
- self.create_new_instances(instance, psd_object)
-
- # Remove the instance from context
- instance.context.remove(instance)
-
- def create_new_instances(self, instance, psd_object):
- asset_doc = instance.data["assetEntity"]
- for layer in psd_object:
- if (
- not layer.is_visible()
- or layer.name.lower() not in self.allowed_group_names
- ):
- continue
-
- has_size = layer.width > 0 and layer.height > 0
- if not has_size:
- self.log.debug((
- "Skipping layer \"{}\" because does "
- "not have any content."
- ).format(layer.name))
- continue
-
- layer_name = layer.name.replace(" ", "_")
- instance_name = subset_name = f"image{layer_name}"
- self.log.info(
- f"Creating new instance with name \"{instance_name}\""
- )
- new_instance = instance.context.create_instance(instance_name)
- for key, value in instance.data.items():
- if key not in self.ignored_instance_data_keys:
- new_instance.data[key] = copy.deepcopy(value)
-
- new_instance.data["label"] = " ".join(
- (new_instance.data["asset"], instance_name)
- )
-
- # Find latest version
- latest_version = self.find_last_version(subset_name, asset_doc)
- version_number = 1
- if latest_version is not None:
- version_number += latest_version
-
- self.log.info(
- "Next version of instance \"{}\" will be {}".format(
- instance_name, version_number
- )
- )
-
- # Set family and subset
- new_instance.data["family"] = self.new_instance_family
- new_instance.data["subset"] = subset_name
- new_instance.data["version"] = version_number
- new_instance.data["latestVersion"] = latest_version
-
- new_instance.data["anatomyData"].update({
- "subset": subset_name,
- "family": self.new_instance_family,
- "version": version_number
- })
-
- # Copy `families` and check if `family` is not in current families
- families = new_instance.data.get("families") or list()
- if families:
- families = list(set(families))
-
- if self.new_instance_family in families:
- families.remove(self.new_instance_family)
- new_instance.data["families"] = families
-
- # Prepare staging dir for new instance
- staging_dir = self.staging_dir(new_instance)
-
- output_filename = "{}.png".format(layer_name)
- output_filepath = os.path.join(staging_dir, output_filename)
- pil_object = layer.composite(viewport=psd_object.viewbox)
- pil_object.save(output_filepath, "PNG")
-
- new_repre = {
- "name": "png",
- "ext": "png",
- "files": output_filename,
- "stagingDir": staging_dir
- }
- self.log.debug(
- "Creating new representation: {}".format(new_repre)
- )
- new_instance.data["representations"] = [new_repre]
-
- def find_last_version(self, subset_name, asset_doc):
- subset_doc = io.find_one({
- "type": "subset",
- "name": subset_name,
- "parent": asset_doc["_id"]
- })
-
- if subset_doc is None:
- self.log.debug("Subset entity does not exist yet.")
- else:
- version_doc = io.find_one(
- {
- "type": "version",
- "parent": subset_doc["_id"]
- },
- sort=[("name", -1)]
- )
- if version_doc:
- return int(version_doc["name"])
- return None
diff --git a/openpype/hosts/standalonepublisher/plugins/publish/extract_thumbnail.py b/openpype/hosts/standalonepublisher/plugins/publish/extract_thumbnail.py
index 23f0b104c8a..3ee2f70809a 100644
--- a/openpype/hosts/standalonepublisher/plugins/publish/extract_thumbnail.py
+++ b/openpype/hosts/standalonepublisher/plugins/publish/extract_thumbnail.py
@@ -2,7 +2,11 @@
import tempfile
import pyblish.api
import openpype.api
-import openpype.lib
+from openpype.lib import (
+ get_ffmpeg_tool_path,
+ get_ffprobe_streams,
+ path_to_subprocess_arg,
+)
class ExtractThumbnailSP(pyblish.api.InstancePlugin):
@@ -34,85 +38,78 @@ def process(self, instance):
if not thumbnail_repre:
return
+ thumbnail_repre.pop("thumbnail")
files = thumbnail_repre.get("files")
if not files:
return
if isinstance(files, list):
- files_len = len(files)
- file = str(files[0])
+ first_filename = str(files[0])
else:
- files_len = 1
- file = files
+ first_filename = files
staging_dir = None
- is_jpeg = False
- if file.endswith(".jpeg") or file.endswith(".jpg"):
- is_jpeg = True
-
- if is_jpeg and files_len == 1:
- # skip if already is single jpeg file
- return
- elif is_jpeg:
- # use first frame as thumbnail if is sequence of jpegs
- full_thumbnail_path = os.path.join(
- thumbnail_repre["stagingDir"], file
- )
- self.log.info(
- "For thumbnail is used file: {}".format(full_thumbnail_path)
- )
+ # Convert to jpeg if not yet
+ full_input_path = os.path.join(
+ thumbnail_repre["stagingDir"], first_filename
+ )
+ self.log.info("input {}".format(full_input_path))
+ with tempfile.NamedTemporaryFile(suffix=".jpg") as tmp:
+ full_thumbnail_path = tmp.name
- else:
- # Convert to jpeg if not yet
- full_input_path = os.path.join(thumbnail_repre["stagingDir"], file)
- self.log.info("input {}".format(full_input_path))
+ self.log.info("output {}".format(full_thumbnail_path))
- full_thumbnail_path = tempfile.mkstemp(suffix=".jpg")[1]
- self.log.info("output {}".format(full_thumbnail_path))
+ instance.context.data["cleanupFullPaths"].append(full_thumbnail_path)
- ffmpeg_path = openpype.lib.get_ffmpeg_tool_path("ffmpeg")
+ ffmpeg_path = get_ffmpeg_tool_path("ffmpeg")
- ffmpeg_args = self.ffmpeg_args or {}
+ ffmpeg_args = self.ffmpeg_args or {}
- jpeg_items = [
- "\"{}\"".format(ffmpeg_path),
- # override file if already exists
- "-y"
- ]
+ jpeg_items = [
+ path_to_subprocess_arg(ffmpeg_path),
+ # override file if already exists
+ "-y"
+ ]
- # add input filters from peresets
- jpeg_items.extend(ffmpeg_args.get("input") or [])
- # input file
- jpeg_items.append("-i \"{}\"".format(full_input_path))
+ # add input filters from peresets
+ jpeg_items.extend(ffmpeg_args.get("input") or [])
+ # input file
+ jpeg_items.extend([
+ "-i", path_to_subprocess_arg(full_input_path),
# extract only single file
- jpeg_items.append("-frames:v 1")
+ "-frames:v", "1",
# Add black background for transparent images
- jpeg_items.append((
- "-filter_complex"
- " \"color=black,format=rgb24[c]"
+ "-filter_complex", (
+ "\"color=black,format=rgb24[c]"
";[c][0]scale2ref[c][i]"
";[c][i]overlay=format=auto:shortest=1,setsar=1\""
- ))
+ ),
+ ])
- jpeg_items.extend(ffmpeg_args.get("output") or [])
+ jpeg_items.extend(ffmpeg_args.get("output") or [])
- # output file
- jpeg_items.append("\"{}\"".format(full_thumbnail_path))
+ # output file
+ jpeg_items.append(path_to_subprocess_arg(full_thumbnail_path))
- subprocess_jpeg = " ".join(jpeg_items)
+ subprocess_jpeg = " ".join(jpeg_items)
- # run subprocess
- self.log.debug("Executing: {}".format(subprocess_jpeg))
- openpype.api.run_subprocess(
- subprocess_jpeg, shell=True, logger=self.log
- )
+ # run subprocess
+ self.log.debug("Executing: {}".format(subprocess_jpeg))
+ openpype.api.run_subprocess(
+ subprocess_jpeg, shell=True, logger=self.log
+ )
# remove thumbnail key from origin repre
- thumbnail_repre.pop("thumbnail")
+ streams = get_ffprobe_streams(full_thumbnail_path)
+ width = height = None
+ for stream in streams:
+ if "width" in stream and "height" in stream:
+ width = stream["width"]
+ height = stream["height"]
+ break
- filename = os.path.basename(full_thumbnail_path)
- staging_dir = staging_dir or os.path.dirname(full_thumbnail_path)
+ staging_dir, filename = os.path.split(full_thumbnail_path)
# create new thumbnail representation
representation = {
@@ -120,12 +117,11 @@ def process(self, instance):
'ext': 'jpg',
'files': filename,
"stagingDir": staging_dir,
- "tags": ["thumbnail"],
+ "tags": ["thumbnail", "delete"],
}
-
- # # add Delete tag when temp file was rendered
- if not is_jpeg:
- representation["tags"].append("delete")
+ if width and height:
+ representation["width"] = width
+ representation["height"] = height
self.log.info(f"New representation {representation}")
instance.data["representations"].append(representation)
diff --git a/openpype/hosts/standalonepublisher/plugins/publish/extract_trim_video_audio.py b/openpype/hosts/standalonepublisher/plugins/publish/extract_trim_video_audio.py
deleted file mode 100644
index c18de5bc1cd..00000000000
--- a/openpype/hosts/standalonepublisher/plugins/publish/extract_trim_video_audio.py
+++ /dev/null
@@ -1,109 +0,0 @@
-import os
-import pyblish.api
-import openpype.api
-
-from pprint import pformat
-
-
-class ExtractTrimVideoAudio(openpype.api.Extractor):
- """Trim with ffmpeg "mov" and "wav" files."""
-
- # must be before `ExtractThumbnailSP`
- order = pyblish.api.ExtractorOrder - 0.01
- label = "Extract Trim Video/Audio"
- hosts = ["standalonepublisher"]
- families = ["clip", "trimming"]
-
- # make sure it is enabled only if at least both families are available
- match = pyblish.api.Subset
-
- # presets
-
- def process(self, instance):
- representation = instance.data.get("representations")
- self.log.debug(f"_ representation: {representation}")
-
- if not representation:
- instance.data["representations"] = list()
-
- # get ffmpet path
- ffmpeg_path = openpype.lib.get_ffmpeg_tool_path("ffmpeg")
-
- # get staging dir
- staging_dir = self.staging_dir(instance)
- self.log.info("Staging dir set to: `{}`".format(staging_dir))
-
- # Generate mov file.
- fps = instance.data["fps"]
- video_file_path = instance.data["editorialSourcePath"]
- extensions = instance.data.get("extensions", [".mov"])
-
- for ext in extensions:
- self.log.info("Processing ext: `{}`".format(ext))
-
- clip_trimed_path = os.path.join(
- staging_dir, instance.data["name"] + ext)
- # # check video file metadata
- # input_data = plib.ffprobe_streams(video_file_path)[0]
- # self.log.debug(f"__ input_data: `{input_data}`")
-
- start = float(instance.data["clipInH"])
- dur = float(instance.data["clipDurationH"])
-
- if ext == ".wav":
- # offset time as ffmpeg is having bug
- start += 0.5
- # remove "review" from families
- instance.data["families"] = [
- fml for fml in instance.data["families"]
- if "trimming" not in fml
- ]
-
- ffmpeg_args = [
- ffmpeg_path,
- "-ss", str(start / fps),
- "-i", video_file_path,
- "-t", str(dur / fps)
- ]
- if ext in [".mov", ".mp4"]:
- ffmpeg_args.extend([
- "-crf", "18",
- "-pix_fmt", "yuv420p"
- ])
- elif ext in ".wav":
- ffmpeg_args.extend([
- "-vn",
- "-acodec", "pcm_s16le",
- "-ar", "48000",
- "-ac", "2"
- ])
-
- # add output path
- ffmpeg_args.append(clip_trimed_path)
-
- joined_args = " ".join(ffmpeg_args)
- self.log.info(f"Processing: {joined_args}")
- openpype.api.run_subprocess(
- ffmpeg_args, logger=self.log
- )
-
- repre = {
- "name": ext[1:],
- "ext": ext[1:],
- "files": os.path.basename(clip_trimed_path),
- "stagingDir": staging_dir,
- "frameStart": int(instance.data["frameStart"]),
- "frameEnd": int(instance.data["frameEnd"]),
- "frameStartFtrack": int(instance.data["frameStartH"]),
- "frameEndFtrack": int(instance.data["frameEndH"]),
- "fps": fps,
- }
-
- if ext in [".mov", ".mp4"]:
- repre.update({
- "thumbnail": True,
- "tags": ["review", "ftrackreview", "delete"]})
-
- instance.data["representations"].append(repre)
-
- self.log.debug(f"Instance data: {pformat(instance.data)}")
diff --git a/openpype/hosts/standalonepublisher/plugins/publish/help/validate_editorial_resources.xml b/openpype/hosts/standalonepublisher/plugins/publish/help/validate_editorial_resources.xml
new file mode 100644
index 00000000000..803de6bf11c
--- /dev/null
+++ b/openpype/hosts/standalonepublisher/plugins/publish/help/validate_editorial_resources.xml
@@ -0,0 +1,17 @@
+
+
+
+Missing source video file
+
+## No attached video file found
+
+Process expects presence of source video file with same name prefix as an editorial file in same folder.
+(example `simple_editorial_setup_Layer1.edl` expects `simple_editorial_setup.mp4` in same folder)
+
+
+### How to repair?
+
+Copy source video file to the folder next to `.edl` file. (On a disk, do not put it into Standalone Publisher.)
+
+
+
diff --git a/openpype/hosts/standalonepublisher/plugins/publish/help/validate_frame_ranges.xml b/openpype/hosts/standalonepublisher/plugins/publish/help/validate_frame_ranges.xml
new file mode 100644
index 00000000000..933df1c7c5c
--- /dev/null
+++ b/openpype/hosts/standalonepublisher/plugins/publish/help/validate_frame_ranges.xml
@@ -0,0 +1,15 @@
+
+
+
+Invalid frame range
+
+## Invalid frame range
+
+Expected duration or '{duration}' frames set in database, workfile contains only '{found}' frames.
+
+### How to repair?
+
+Modify configuration in the database or tweak frame range in the workfile.
+
+
+
\ No newline at end of file
diff --git a/openpype/hosts/standalonepublisher/plugins/publish/help/validate_shot_duplicates.xml b/openpype/hosts/standalonepublisher/plugins/publish/help/validate_shot_duplicates.xml
new file mode 100644
index 00000000000..77b8727162e
--- /dev/null
+++ b/openpype/hosts/standalonepublisher/plugins/publish/help/validate_shot_duplicates.xml
@@ -0,0 +1,15 @@
+
+
+
+Duplicate shots
+
+## Duplicate shot names
+
+Process contains duplicated shot names '{duplicates_str}'.
+
+### How to repair?
+
+Remove shot duplicates.
+
+
+
\ No newline at end of file
diff --git a/openpype/hosts/standalonepublisher/plugins/publish/help/validate_simple_texture_naming.xml b/openpype/hosts/standalonepublisher/plugins/publish/help/validate_simple_texture_naming.xml
new file mode 100644
index 00000000000..b65d274fe53
--- /dev/null
+++ b/openpype/hosts/standalonepublisher/plugins/publish/help/validate_simple_texture_naming.xml
@@ -0,0 +1,17 @@
+
+
+
+Invalid texture name
+
+## Invalid file name
+
+Submitted file has invalid name:
+'{invalid_file}'
+
+### How to repair?
+
+ Texture file must adhere to naming conventions for Unreal:
+ T_{asset}_*.ext
+
+
+
\ No newline at end of file
diff --git a/openpype/hosts/standalonepublisher/plugins/publish/help/validate_sources.xml b/openpype/hosts/standalonepublisher/plugins/publish/help/validate_sources.xml
new file mode 100644
index 00000000000..d527d2173ec
--- /dev/null
+++ b/openpype/hosts/standalonepublisher/plugins/publish/help/validate_sources.xml
@@ -0,0 +1,16 @@
+
+
+
+Files not found
+
+## Source files not found
+
+Process contains duplicated shot names:
+'{files_not_found}'
+
+### How to repair?
+
+Add missing files or run Publish again to collect new publishable files.
+
+
+
\ No newline at end of file
diff --git a/openpype/hosts/standalonepublisher/plugins/publish/help/validate_task_existence.xml b/openpype/hosts/standalonepublisher/plugins/publish/help/validate_task_existence.xml
new file mode 100644
index 00000000000..a943f560d08
--- /dev/null
+++ b/openpype/hosts/standalonepublisher/plugins/publish/help/validate_task_existence.xml
@@ -0,0 +1,16 @@
+
+
+
+Task not found
+
+## Task not found in database
+
+Process contains tasks that don't exist in database:
+'{task_not_found}'
+
+### How to repair?
+
+Remove set task or add task into database into proper place.
+
+
+
\ No newline at end of file
diff --git a/openpype/hosts/standalonepublisher/plugins/publish/help/validate_texture_batch.xml b/openpype/hosts/standalonepublisher/plugins/publish/help/validate_texture_batch.xml
new file mode 100644
index 00000000000..a645df8d02f
--- /dev/null
+++ b/openpype/hosts/standalonepublisher/plugins/publish/help/validate_texture_batch.xml
@@ -0,0 +1,15 @@
+
+
+
+No texture files found
+
+## Batch doesn't contain texture files
+
+Batch must contain at least one texture file.
+
+### How to repair?
+
+Add texture file to the batch or check name if it follows naming convention to match texture files to the batch.
+
+
+
\ No newline at end of file
diff --git a/openpype/hosts/standalonepublisher/plugins/publish/help/validate_texture_has_workfile.xml b/openpype/hosts/standalonepublisher/plugins/publish/help/validate_texture_has_workfile.xml
new file mode 100644
index 00000000000..077987a96de
--- /dev/null
+++ b/openpype/hosts/standalonepublisher/plugins/publish/help/validate_texture_has_workfile.xml
@@ -0,0 +1,15 @@
+
+
+
+No workfile found
+
+## Batch should contain workfile
+
+It is expected that published contains workfile that served as a source for textures.
+
+### How to repair?
+
+Add workfile to the batch, or disable this validator if you do not want workfile published.
+
+
+
\ No newline at end of file
diff --git a/openpype/hosts/standalonepublisher/plugins/publish/help/validate_texture_name.xml b/openpype/hosts/standalonepublisher/plugins/publish/help/validate_texture_name.xml
new file mode 100644
index 00000000000..26109177363
--- /dev/null
+++ b/openpype/hosts/standalonepublisher/plugins/publish/help/validate_texture_name.xml
@@ -0,0 +1,32 @@
+
+
+
+Asset name not found
+
+## Couldn't parse asset name from a file
+
+Unable to parse asset name from '{file_name}'. File name doesn't match configured naming convention.
+
+### How to repair?
+
+Check Settings: project_settings/standalonepublisher/publish/CollectTextures for naming convention.
+
+
+### __Detailed Info__ (optional)
+
+This error happens when parsing cannot figure out name of asset texture files belong under.
+
+
+
+Missing keys
+
+## Texture file name is missing some required keys
+
+Texture '{file_name}' is missing values for {missing_str} keys.
+
+### How to repair?
+
+Fix name of texture file and Publish again.
+
+
+
diff --git a/openpype/hosts/standalonepublisher/plugins/publish/help/validate_texture_versions.xml b/openpype/hosts/standalonepublisher/plugins/publish/help/validate_texture_versions.xml
new file mode 100644
index 00000000000..1e536e604f2
--- /dev/null
+++ b/openpype/hosts/standalonepublisher/plugins/publish/help/validate_texture_versions.xml
@@ -0,0 +1,35 @@
+
+
+
+Texture version
+
+## Texture version mismatch with workfile
+
+Workfile '{file_name}' version doesn't match with '{version}' of a texture.
+
+### How to repair?
+
+Rename either workfile or texture to contain matching versions
+
+
+### __Detailed Info__ (optional)
+
+This might happen if you are trying to publish textures for older version of workfile (or the other way).
+(Eg. publishing 'workfile_v001' and 'texture_file_v002')
+
+
+
+Too many versions
+
+## Too many versions published at same time
+
+It is currently expected to publish only batch with single version.
+
+Found {found} versions.
+
+### How to repair?
+
+Please remove files with different version and split publishing into multiple steps.
+
+
+
diff --git a/openpype/hosts/standalonepublisher/plugins/publish/help/validate_texture_workfiles.xml b/openpype/hosts/standalonepublisher/plugins/publish/help/validate_texture_workfiles.xml
new file mode 100644
index 00000000000..8187eb0bc8a
--- /dev/null
+++ b/openpype/hosts/standalonepublisher/plugins/publish/help/validate_texture_workfiles.xml
@@ -0,0 +1,23 @@
+
+
+
+No secondary workfile
+
+## No secondary workfile found
+
+Current process expects that primary workfile (for example with a extension '{extension}') will contain also 'secondary' workfile.
+
+Secondary workfile for '{file_name}' wasn't found.
+
+### How to repair?
+
+Attach secondary workfile or disable this validator and Publish again.
+
+
+### __Detailed Info__ (optional)
+
+This process was implemented for a possible use case of first workfile coming from Mari, secondary workfile for textures from Substance.
+Publish should contain both if primary workfile is present.
+
+
+
diff --git a/openpype/hosts/standalonepublisher/plugins/publish/validate_editorial_resources.py b/openpype/hosts/standalonepublisher/plugins/publish/validate_editorial_resources.py
index 6759b87ceba..3d2b6d04ad4 100644
--- a/openpype/hosts/standalonepublisher/plugins/publish/validate_editorial_resources.py
+++ b/openpype/hosts/standalonepublisher/plugins/publish/validate_editorial_resources.py
@@ -1,5 +1,8 @@
import pyblish.api
-import openpype.api
+from openpype.pipeline.publish import (
+ ValidateContentsOrder,
+ PublishXmlValidationError,
+)
class ValidateEditorialResources(pyblish.api.InstancePlugin):
@@ -12,12 +15,14 @@ class ValidateEditorialResources(pyblish.api.InstancePlugin):
# make sure it is enabled only if at least both families are available
match = pyblish.api.Subset
- order = openpype.api.ValidateContentsOrder
+ order = ValidateContentsOrder
def process(self, instance):
self.log.debug(
f"Instance: {instance}, Families: "
f"{[instance.data['family']] + instance.data['families']}")
check_file = instance.data["editorialSourcePath"]
- msg = f"Missing \"{check_file}\"."
- assert check_file, msg
+ msg = "Missing source video file."
+
+ if not check_file:
+ raise PublishXmlValidationError(self, msg)
diff --git a/openpype/hosts/standalonepublisher/plugins/publish/validate_frame_ranges.py b/openpype/hosts/standalonepublisher/plugins/publish/validate_frame_ranges.py
index 943cb73b98a..074c62ea0e9 100644
--- a/openpype/hosts/standalonepublisher/plugins/publish/validate_frame_ranges.py
+++ b/openpype/hosts/standalonepublisher/plugins/publish/validate_frame_ranges.py
@@ -1,8 +1,12 @@
import re
import pyblish.api
-import openpype.api
-from openpype import lib
+
+from openpype.pipeline.context_tools import get_current_project_asset
+from openpype.pipeline.publish import (
+ ValidateContentsOrder,
+ PublishXmlValidationError,
+)
class ValidateFrameRange(pyblish.api.InstancePlugin):
@@ -11,7 +15,7 @@ class ValidateFrameRange(pyblish.api.InstancePlugin):
label = "Validate Frame Range"
hosts = ["standalonepublisher"]
families = ["render"]
- order = openpype.api.ValidateContentsOrder
+ order = ValidateContentsOrder
optional = True
# published data might be sequence (.mov, .mp4) in that counting files
@@ -25,7 +29,8 @@ def process(self, instance):
for pattern in self.skip_timelines_check):
self.log.info("Skipping for {} task".format(instance.data["task"]))
- asset_data = lib.get_asset(instance.data["asset"])["data"]
+ # TODO repace query with using 'instance.data["assetEntity"]'
+ asset_data = get_current_project_asset(instance.data["asset"])["data"]
frame_start = asset_data["frameStart"]
frame_end = asset_data["frameEnd"]
handle_start = asset_data["handleStart"]
@@ -48,9 +53,15 @@ def process(self, instance):
files = [files]
frames = len(files)
- err_msg = "Frame duration from DB:'{}' ". format(int(duration)) +\
- " doesn't match number of files:'{}'".format(frames) +\
- " Please change frame range for Asset or limit no. of files"
- assert frames == duration, err_msg
+ msg = "Frame duration from DB:'{}' ". format(int(duration)) +\
+ " doesn't match number of files:'{}'".format(frames) +\
+ " Please change frame range for Asset or limit no. of files"
+
+ formatting_data = {"duration": duration,
+ "found": frames}
+ if frames != duration:
+ raise PublishXmlValidationError(self, msg,
+ formatting_data=formatting_data)
- self.log.debug("Valid ranges {} - {}".format(int(duration), frames))
+ self.log.debug("Valid ranges expected '{}' - found '{}'".
+ format(int(duration), frames))
diff --git a/openpype/hosts/standalonepublisher/plugins/publish/validate_shot_duplicates.py b/openpype/hosts/standalonepublisher/plugins/publish/validate_shot_duplicates.py
index 85ec9379ce7..df04ae3b660 100644
--- a/openpype/hosts/standalonepublisher/plugins/publish/validate_shot_duplicates.py
+++ b/openpype/hosts/standalonepublisher/plugins/publish/validate_shot_duplicates.py
@@ -1,5 +1,9 @@
import pyblish.api
-import openpype.api
+
+from openpype.pipeline.publish import (
+ ValidateContentsOrder,
+ PublishXmlValidationError,
+)
class ValidateShotDuplicates(pyblish.api.ContextPlugin):
@@ -7,7 +11,7 @@ class ValidateShotDuplicates(pyblish.api.ContextPlugin):
label = "Validate Shot Duplicates"
hosts = ["standalonepublisher"]
- order = openpype.api.ValidateContentsOrder
+ order = ValidateContentsOrder
def process(self, context):
shot_names = []
@@ -20,4 +24,8 @@ def process(self, context):
shot_names.append(name)
msg = "There are duplicate shot names:\n{}".format(duplicate_names)
- assert not duplicate_names, msg
+
+ formatting_data = {"duplicates_str": ','.join(duplicate_names)}
+ if duplicate_names:
+ raise PublishXmlValidationError(self, msg,
+ formatting_data=formatting_data)
diff --git a/openpype/hosts/standalonepublisher/plugins/publish/validate_simple_unreal_texture_naming.py b/openpype/hosts/standalonepublisher/plugins/publish/validate_simple_unreal_texture_naming.py
new file mode 100644
index 00000000000..c123bef4f89
--- /dev/null
+++ b/openpype/hosts/standalonepublisher/plugins/publish/validate_simple_unreal_texture_naming.py
@@ -0,0 +1,29 @@
+# -*- coding: utf-8 -*-
+"""Validator for correct file naming."""
+import re
+import pyblish.api
+
+from openpype.pipeline.publish import (
+ ValidateContentsOrder,
+ PublishXmlValidationError,
+)
+
+
+class ValidateSimpleUnrealTextureNaming(pyblish.api.InstancePlugin):
+ label = "Validate Unreal Texture Names"
+ hosts = ["standalonepublisher"]
+ families = ["simpleUnrealTexture"]
+ order = ValidateContentsOrder
+ regex = "^T_{asset}.*"
+
+ def process(self, instance):
+ file_name = instance.data.get("originalBasename")
+ self.log.info(file_name)
+ pattern = self.regex.format(asset=instance.data.get("asset"))
+ if not re.match(pattern, file_name):
+ msg = f"Invalid file name {file_name}"
+ raise PublishXmlValidationError(
+ self, msg, formatting_data={
+ "invalid_file": file_name,
+ "asset": instance.data.get("asset")
+ })
diff --git a/openpype/hosts/standalonepublisher/plugins/publish/validate_sources.py b/openpype/hosts/standalonepublisher/plugins/publish/validate_sources.py
index eec675e97f6..1782f53de25 100644
--- a/openpype/hosts/standalonepublisher/plugins/publish/validate_sources.py
+++ b/openpype/hosts/standalonepublisher/plugins/publish/validate_sources.py
@@ -1,7 +1,11 @@
+import os
+
import pyblish.api
-import openpype.api
-import os
+from openpype.pipeline.publish import (
+ ValidateContentsOrder,
+ PublishXmlValidationError,
+)
class ValidateSources(pyblish.api.InstancePlugin):
@@ -11,8 +15,7 @@ class ValidateSources(pyblish.api.InstancePlugin):
got deleted between starting of SP and now.
"""
-
- order = openpype.api.ValidateContentsOrder
+ order = ValidateContentsOrder
label = "Check source files"
optional = True # only for unforeseeable cases
@@ -22,6 +25,7 @@ class ValidateSources(pyblish.api.InstancePlugin):
def process(self, instance):
self.log.info("instance {}".format(instance.data))
+ missing_files = set()
for repre in instance.data.get("representations") or []:
files = []
if isinstance(repre["files"], str):
@@ -34,4 +38,10 @@ def process(self, instance):
file_name)
if not os.path.exists(source_file):
- raise ValueError("File {} not found".format(source_file))
+ missing_files.add(source_file)
+
+ msg = "Files '{}' not found".format(','.join(missing_files))
+ formatting_data = {"files_not_found": ' - {}'.join(missing_files)}
+ if missing_files:
+ raise PublishXmlValidationError(self, msg,
+ formatting_data=formatting_data)
diff --git a/openpype/hosts/standalonepublisher/plugins/publish/validate_task_existence.py b/openpype/hosts/standalonepublisher/plugins/publish/validate_task_existence.py
index e3b2ae16467..19ea1a47780 100644
--- a/openpype/hosts/standalonepublisher/plugins/publish/validate_task_existence.py
+++ b/openpype/hosts/standalonepublisher/plugins/publish/validate_task_existence.py
@@ -1,5 +1,7 @@
import pyblish.api
-from avalon import io
+
+from openpype.client import get_assets
+from openpype.pipeline import PublishXmlValidationError
class ValidateTaskExistence(pyblish.api.ContextPlugin):
@@ -16,15 +18,11 @@ def process(self, context):
for instance in context:
asset_names.add(instance.data["asset"])
- asset_docs = io.find(
- {
- "type": "asset",
- "name": {"$in": list(asset_names)}
- },
- {
- "name": 1,
- "data.tasks": 1
- }
+ project_name = context.data["projectEntity"]["name"]
+ asset_docs = get_assets(
+ project_name,
+ asset_names=asset_names,
+ fields=["name", "data.tasks"]
)
tasks_by_asset_names = {}
for asset_doc in asset_docs:
@@ -53,4 +51,9 @@ def process(self, context):
"Asset: \"{}\" Task: \"{}\"".format(*missing_pair)
)
- raise AssertionError(msg.format("\n".join(pair_msgs)))
+ msg = msg.format("\n".join(pair_msgs))
+
+ formatting_data = {"task_not_found": ' - {}'.join(pair_msgs)}
+ if pair_msgs:
+ raise PublishXmlValidationError(self, msg,
+ formatting_data=formatting_data)
diff --git a/openpype/hosts/standalonepublisher/plugins/publish/validate_texture_batch.py b/openpype/hosts/standalonepublisher/plugins/publish/validate_texture_batch.py
index d592a4a0599..44f69e48f7d 100644
--- a/openpype/hosts/standalonepublisher/plugins/publish/validate_texture_batch.py
+++ b/openpype/hosts/standalonepublisher/plugins/publish/validate_texture_batch.py
@@ -1,5 +1,9 @@
import pyblish.api
-import openpype.api
+
+from openpype.pipeline.publish import (
+ ValidateContentsOrder,
+ PublishXmlValidationError,
+)
class ValidateTextureBatch(pyblish.api.InstancePlugin):
@@ -7,7 +11,7 @@ class ValidateTextureBatch(pyblish.api.InstancePlugin):
label = "Validate Texture Presence"
hosts = ["standalonepublisher"]
- order = openpype.api.ValidateContentsOrder
+ order = ValidateContentsOrder
families = ["texture_batch_workfile"]
optional = False
@@ -15,8 +19,10 @@ def process(self, instance):
present = False
for instance in instance.context:
if instance.data["family"] == "textures":
- self.log.info("Some textures present.")
+ self.log.info("At least some textures present.")
return
- assert present, "No textures found in published batch!"
+ msg = "No textures found in published batch!"
+ if not present:
+ raise PublishXmlValidationError(self, msg)
diff --git a/openpype/hosts/standalonepublisher/plugins/publish/validate_texture_has_workfile.py b/openpype/hosts/standalonepublisher/plugins/publish/validate_texture_has_workfile.py
index 7cd540668cc..f489d37f595 100644
--- a/openpype/hosts/standalonepublisher/plugins/publish/validate_texture_has_workfile.py
+++ b/openpype/hosts/standalonepublisher/plugins/publish/validate_texture_has_workfile.py
@@ -1,5 +1,9 @@
import pyblish.api
-import openpype.api
+
+from openpype.pipeline.publish import (
+ ValidateContentsOrder,
+ PublishXmlValidationError,
+)
class ValidateTextureHasWorkfile(pyblish.api.InstancePlugin):
@@ -10,11 +14,13 @@ class ValidateTextureHasWorkfile(pyblish.api.InstancePlugin):
"""
label = "Validate Texture Has Workfile"
hosts = ["standalonepublisher"]
- order = openpype.api.ValidateContentsOrder
+ order = ValidateContentsOrder
families = ["textures"]
optional = True
def process(self, instance):
wfile = instance.data["versionData"].get("workfile")
- assert wfile, "Textures are missing attached workfile"
+ msg = "Textures are missing attached workfile"
+ if not wfile:
+ raise PublishXmlValidationError(self, msg)
diff --git a/openpype/hosts/standalonepublisher/plugins/publish/validate_texture_name.py b/openpype/hosts/standalonepublisher/plugins/publish/validate_texture_name.py
index f210be36315..22f4a0eafc3 100644
--- a/openpype/hosts/standalonepublisher/plugins/publish/validate_texture_name.py
+++ b/openpype/hosts/standalonepublisher/plugins/publish/validate_texture_name.py
@@ -1,13 +1,16 @@
import pyblish.api
-import openpype.api
+from openpype.pipeline.publish import (
+ ValidateContentsOrder,
+ PublishXmlValidationError,
+)
class ValidateTextureBatchNaming(pyblish.api.InstancePlugin):
"""Validates that all instances had properly formatted name."""
label = "Validate Texture Batch Naming"
hosts = ["standalonepublisher"]
- order = openpype.api.ValidateContentsOrder
+ order = ValidateContentsOrder
families = ["texture_batch_workfile", "textures"]
optional = False
@@ -16,12 +19,16 @@ def process(self, instance):
if isinstance(file_name, list):
file_name = file_name[0]
- msg = "Couldnt find asset name in '{}'\n".format(file_name) + \
+ msg = "Couldn't find asset name in '{}'\n".format(file_name) + \
"File name doesn't follow configured pattern.\n" + \
"Please rename the file."
- assert "NOT_AVAIL" not in instance.data["asset_build"], msg
- instance.data.pop("asset_build")
+ formatting_data = {"file_name": file_name}
+ if "NOT_AVAIL" in instance.data["asset_build"]:
+ raise PublishXmlValidationError(self, msg,
+ formatting_data=formatting_data)
+
+ instance.data.pop("asset_build") # not needed anymore
if instance.data["family"] == "textures":
file_name = instance.data["representations"][0]["files"][0]
@@ -47,4 +54,10 @@ def _check_proper_collected(self, versionData, file_name):
"Name of the texture file doesn't match expected pattern.\n" + \
"Please rename file(s) {}".format(file_name)
- assert not missing_key_values, msg
+ missing_str = ','.join(["'{}'".format(key)
+ for key in missing_key_values])
+ formatting_data = {"file_name": file_name,
+ "missing_str": missing_str}
+ if missing_key_values:
+ raise PublishXmlValidationError(self, msg, key="missing_values",
+ formatting_data=formatting_data)
diff --git a/openpype/hosts/standalonepublisher/plugins/publish/validate_texture_versions.py b/openpype/hosts/standalonepublisher/plugins/publish/validate_texture_versions.py
index 90d0e8e512b..dab160d537d 100644
--- a/openpype/hosts/standalonepublisher/plugins/publish/validate_texture_versions.py
+++ b/openpype/hosts/standalonepublisher/plugins/publish/validate_texture_versions.py
@@ -1,5 +1,9 @@
import pyblish.api
-import openpype.api
+
+from openpype.pipeline.publish import (
+ ValidateContentsOrder,
+ PublishXmlValidationError,
+)
class ValidateTextureBatchVersions(pyblish.api.InstancePlugin):
@@ -12,7 +16,7 @@ class ValidateTextureBatchVersions(pyblish.api.InstancePlugin):
"""
label = "Validate Texture Batch Versions"
hosts = ["standalonepublisher"]
- order = openpype.api.ValidateContentsOrder
+ order = ValidateContentsOrder
families = ["textures"]
optional = False
@@ -25,14 +29,21 @@ def process(self, instance):
self.log.info("No workfile present for textures")
return
- msg = "Not matching version: texture v{:03d} - workfile {}"
- assert version_str in wfile, \
+ if version_str not in wfile:
+ msg = "Not matching version: texture v{:03d} - workfile {}"
msg.format(
instance.data["version"], wfile
)
+ raise PublishXmlValidationError(self, msg)
present_versions = set()
for instance in instance.context:
present_versions.add(instance.data["version"])
- assert len(present_versions) == 1, "Too many versions in a batch!"
+ if len(present_versions) != 1:
+ msg = "Too many versions in a batch!"
+ found = ','.join(["'{}'".format(val) for val in present_versions])
+ formatting_data = {"found": found}
+
+ raise PublishXmlValidationError(self, msg, key="too_many",
+ formatting_data=formatting_data)
diff --git a/openpype/hosts/standalonepublisher/plugins/publish/validate_texture_workfiles.py b/openpype/hosts/standalonepublisher/plugins/publish/validate_texture_workfiles.py
index 25bb5aea4ae..56ea82f6b69 100644
--- a/openpype/hosts/standalonepublisher/plugins/publish/validate_texture_workfiles.py
+++ b/openpype/hosts/standalonepublisher/plugins/publish/validate_texture_workfiles.py
@@ -1,16 +1,20 @@
import pyblish.api
-import openpype.api
+
+from openpype.pipeline.publish import (
+ ValidateContentsOrder,
+ PublishXmlValidationError,
+)
class ValidateTextureBatchWorkfiles(pyblish.api.InstancePlugin):
"""Validates that textures workfile has collected resources (optional).
- Collected recourses means secondary workfiles (in most cases).
+ Collected resources means secondary workfiles (in most cases).
"""
label = "Validate Texture Workfile Has Resources"
hosts = ["standalonepublisher"]
- order = openpype.api.ValidateContentsOrder
+ order = ValidateContentsOrder
families = ["texture_batch_workfile"]
optional = True
@@ -24,6 +28,13 @@ def process(self, instance):
self.log.warning("Only secondary workfile present!")
return
- msg = "No secondary workfiles present for workfile {}".\
- format(instance.data["name"])
- assert instance.data.get("resources"), msg
+ if not instance.data.get("resources"):
+ msg = "No secondary workfile present for workfile '{}'". \
+ format(instance.data["name"])
+ ext = self.main_workfile_extensions[0]
+ formatting_data = {"file_name": instance.data["name"],
+ "extension": ext}
+
+ raise PublishXmlValidationError(self, msg,
+ formatting_data=formatting_data
+ )
diff --git a/openpype/hosts/testhost/README.md b/openpype/hosts/testhost/README.md
deleted file mode 100644
index f69e02a3b36..00000000000
--- a/openpype/hosts/testhost/README.md
+++ /dev/null
@@ -1,16 +0,0 @@
-# What is `testhost`
-Host `testhost` was created to fake running host for testing of publisher.
-
-Does not have any proper launch mechanism at the moment. There is python script `./run_publish.py` which will show publisher window. The script requires to set few variables to run. Execution will register host `testhost`, register global publish plugins and register creator and publish plugins from `./plugins`.
-
-## Data
-Created instances and context data are stored into json files inside `./api` folder. Can be easily modified to save them to a different place.
-
-## Plugins
-Test host has few plugins to be able test publishing.
-
-### Creators
-They are just example plugins using functions from `api` to create/remove/update data. One of them is auto creator which means that is triggered on each reset of create context. Others are manual creators both creating the same family.
-
-### Publishers
-Collectors are example plugin to use `get_attribute_defs` to define attributes for specific families or for context. Validators are to test `PublishValidationError`.
diff --git a/openpype/hosts/testhost/api/__init__.py b/openpype/hosts/testhost/api/__init__.py
deleted file mode 100644
index 7840b25892d..00000000000
--- a/openpype/hosts/testhost/api/__init__.py
+++ /dev/null
@@ -1,43 +0,0 @@
-import os
-import logging
-import pyblish.api
-import avalon.api
-from openpype.pipeline import BaseCreator
-
-from .pipeline import (
- ls,
- list_instances,
- update_instances,
- remove_instances,
- get_context_data,
- update_context_data,
- get_context_title
-)
-
-
-HOST_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
-PLUGINS_DIR = os.path.join(HOST_DIR, "plugins")
-PUBLISH_PATH = os.path.join(PLUGINS_DIR, "publish")
-CREATE_PATH = os.path.join(PLUGINS_DIR, "create")
-
-log = logging.getLogger(__name__)
-
-
-def install():
- log.info("OpenPype - Installing TestHost integration")
- pyblish.api.register_host("testhost")
- pyblish.api.register_plugin_path(PUBLISH_PATH)
- avalon.api.register_plugin_path(BaseCreator, CREATE_PATH)
-
-
-__all__ = (
- "ls",
- "list_instances",
- "update_instances",
- "remove_instances",
- "get_context_data",
- "update_context_data",
- "get_context_title",
-
- "install"
-)
diff --git a/openpype/hosts/testhost/api/context.json b/openpype/hosts/testhost/api/context.json
deleted file mode 100644
index 0967ef424bc..00000000000
--- a/openpype/hosts/testhost/api/context.json
+++ /dev/null
@@ -1 +0,0 @@
-{}
diff --git a/openpype/hosts/testhost/api/instances.json b/openpype/hosts/testhost/api/instances.json
deleted file mode 100644
index 84021eff91d..00000000000
--- a/openpype/hosts/testhost/api/instances.json
+++ /dev/null
@@ -1,108 +0,0 @@
-[
- {
- "id": "pyblish.avalon.instance",
- "active": true,
- "family": "test",
- "subset": "testMyVariant",
- "version": 1,
- "asset": "sq01_sh0010",
- "task": "Compositing",
- "variant": "myVariant",
- "uuid": "a485f148-9121-46a5-8157-aa64df0fb449",
- "creator_attributes": {
- "number_key": 10,
- "ha": 10
- },
- "publish_attributes": {
- "CollectFtrackApi": {
- "add_ftrack_family": false
- }
- },
- "creator_identifier": "test_one"
- },
- {
- "id": "pyblish.avalon.instance",
- "active": true,
- "family": "test",
- "subset": "testMyVariant2",
- "version": 1,
- "asset": "sq01_sh0010",
- "task": "Compositing",
- "variant": "myVariant2",
- "uuid": "a485f148-9121-46a5-8157-aa64df0fb444",
- "creator_attributes": {},
- "publish_attributes": {
- "CollectFtrackApi": {
- "add_ftrack_family": true
- }
- },
- "creator_identifier": "test_one"
- },
- {
- "id": "pyblish.avalon.instance",
- "active": true,
- "family": "test",
- "subset": "testMain",
- "version": 1,
- "asset": "sq01_sh0010",
- "task": "Compositing",
- "variant": "Main",
- "uuid": "3607bc95-75f6-4648-a58d-e699f413d09f",
- "creator_attributes": {},
- "publish_attributes": {
- "CollectFtrackApi": {
- "add_ftrack_family": true
- }
- },
- "creator_identifier": "test_two"
- },
- {
- "id": "pyblish.avalon.instance",
- "active": true,
- "family": "test",
- "subset": "testMain2",
- "version": 1,
- "asset": "sq01_sh0020",
- "task": "Compositing",
- "variant": "Main2",
- "uuid": "4ccf56f6-9982-4837-967c-a49695dbe8eb",
- "creator_attributes": {},
- "publish_attributes": {
- "CollectFtrackApi": {
- "add_ftrack_family": true
- }
- },
- "creator_identifier": "test_two"
- },
- {
- "id": "pyblish.avalon.instance",
- "family": "test_three",
- "subset": "test_threeMain2",
- "active": true,
- "version": 1,
- "asset": "sq01_sh0020",
- "task": "Compositing",
- "variant": "Main2",
- "uuid": "4ccf56f6-9982-4837-967c-a49695dbe8ec",
- "creator_attributes": {},
- "publish_attributes": {
- "CollectFtrackApi": {
- "add_ftrack_family": true
- }
- }
- },
- {
- "id": "pyblish.avalon.instance",
- "family": "workfile",
- "subset": "workfileMain",
- "active": true,
- "creator_identifier": "workfile",
- "version": 1,
- "asset": "Alpaca_01",
- "task": "modeling",
- "variant": "Main",
- "uuid": "7c9ddfc7-9f9c-4c1c-b233-38c966735fb6",
- "creator_attributes": {},
- "publish_attributes": {}
- }
-]
\ No newline at end of file
diff --git a/openpype/hosts/testhost/api/pipeline.py b/openpype/hosts/testhost/api/pipeline.py
deleted file mode 100644
index 49f1d3f33d7..00000000000
--- a/openpype/hosts/testhost/api/pipeline.py
+++ /dev/null
@@ -1,156 +0,0 @@
-import os
-import json
-
-
-class HostContext:
- instances_json_path = None
- context_json_path = None
-
- @classmethod
- def get_context_title(cls):
- project_name = os.environ.get("AVALON_PROJECT")
- if not project_name:
- return "TestHost"
-
- asset_name = os.environ.get("AVALON_ASSET")
- if not asset_name:
- return project_name
-
- from avalon import io
-
- asset_doc = io.find_one(
- {"type": "asset", "name": asset_name},
- {"data.parents": 1}
- )
- parents = asset_doc.get("data", {}).get("parents") or []
-
- hierarchy = [project_name]
- hierarchy.extend(parents)
- hierarchy.append("{} ".format(asset_name))
- task_name = os.environ.get("AVALON_TASK")
- if task_name:
- hierarchy.append(task_name)
-
- return "/".join(hierarchy)
-
- @classmethod
- def get_current_dir_filepath(cls, filename):
- return os.path.join(
- os.path.dirname(os.path.abspath(__file__)),
- filename
- )
-
- @classmethod
- def get_instances_json_path(cls):
- if cls.instances_json_path is None:
- cls.instances_json_path = cls.get_current_dir_filepath(
- "instances.json"
- )
- return cls.instances_json_path
-
- @classmethod
- def get_context_json_path(cls):
- if cls.context_json_path is None:
- cls.context_json_path = cls.get_current_dir_filepath(
- "context.json"
- )
- return cls.context_json_path
-
- @classmethod
- def add_instance(cls, instance):
- instances = cls.get_instances()
- instances.append(instance)
- cls.save_instances(instances)
-
- @classmethod
- def save_instances(cls, instances):
- json_path = cls.get_instances_json_path()
- with open(json_path, "w") as json_stream:
- json.dump(instances, json_stream, indent=4)
-
- @classmethod
- def get_instances(cls):
- json_path = cls.get_instances_json_path()
- if not os.path.exists(json_path):
- instances = []
- with open(json_path, "w") as json_stream:
- json.dump(json_stream, instances)
- else:
- with open(json_path, "r") as json_stream:
- instances = json.load(json_stream)
- return instances
-
- @classmethod
- def get_context_data(cls):
- json_path = cls.get_context_json_path()
- if not os.path.exists(json_path):
- data = {}
- with open(json_path, "w") as json_stream:
- json.dump(data, json_stream)
- else:
- with open(json_path, "r") as json_stream:
- data = json.load(json_stream)
- return data
-
- @classmethod
- def save_context_data(cls, data):
- json_path = cls.get_context_json_path()
- with open(json_path, "w") as json_stream:
- json.dump(data, json_stream, indent=4)
-
-
-def ls():
- return []
-
-
-def list_instances():
- return HostContext.get_instances()
-
-
-def update_instances(update_list):
- updated_instances = {}
- for instance, _changes in update_list:
- updated_instances[instance.id] = instance.data_to_store()
-
- instances = HostContext.get_instances()
- for instance_data in instances:
- instance_id = instance_data["uuid"]
- if instance_id in updated_instances:
- new_instance_data = updated_instances[instance_id]
- old_keys = set(instance_data.keys())
- new_keys = set(new_instance_data.keys())
- instance_data.update(new_instance_data)
- for key in (old_keys - new_keys):
- instance_data.pop(key)
-
- HostContext.save_instances(instances)
-
-
-def remove_instances(instances):
- if not isinstance(instances, (tuple, list)):
- instances = [instances]
-
- current_instances = HostContext.get_instances()
- for instance in instances:
- instance_id = instance.data["uuid"]
- found_idx = None
- for idx, _instance in enumerate(current_instances):
- if instance_id == _instance["uuid"]:
- found_idx = idx
- break
-
- if found_idx is not None:
- current_instances.pop(found_idx)
- HostContext.save_instances(current_instances)
-
-
-def get_context_data():
- return HostContext.get_context_data()
-
-
-def update_context_data(data, changes):
- HostContext.save_context_data(data)
-
-
-def get_context_title():
- return HostContext.get_context_title()
diff --git a/openpype/hosts/testhost/plugins/create/auto_creator.py b/openpype/hosts/testhost/plugins/create/auto_creator.py
deleted file mode 100644
index 0690164ae54..00000000000
--- a/openpype/hosts/testhost/plugins/create/auto_creator.py
+++ /dev/null
@@ -1,74 +0,0 @@
-from openpype.hosts.testhost.api import pipeline
-from openpype.pipeline import (
- AutoCreator,
- CreatedInstance,
- lib
-)
-from avalon import io
-
-
-class MyAutoCreator(AutoCreator):
- identifier = "workfile"
- family = "workfile"
-
- def get_attribute_defs(self):
- output = [
- lib.NumberDef("number_key", label="Number")
- ]
- return output
-
- def collect_instances(self):
- for instance_data in pipeline.list_instances():
- creator_id = instance_data.get("creator_identifier")
- if creator_id == self.identifier:
- subset_name = instance_data["subset"]
- instance = CreatedInstance(
- self.family, subset_name, instance_data, self
- )
- self._add_instance_to_context(instance)
-
- def update_instances(self, update_list):
- pipeline.update_instances(update_list)
-
- def create(self, options=None):
- existing_instance = None
- for instance in self.create_context.instances:
- if instance.family == self.family:
- existing_instance = instance
- break
-
- variant = "Main"
- project_name = io.Session["AVALON_PROJECT"]
- asset_name = io.Session["AVALON_ASSET"]
- task_name = io.Session["AVALON_TASK"]
- host_name = io.Session["AVALON_APP"]
-
- if existing_instance is None:
- asset_doc = io.find_one({"type": "asset", "name": asset_name})
- subset_name = self.get_subset_name(
- variant, task_name, asset_doc, project_name, host_name
- )
- data = {
- "asset": asset_name,
- "task": task_name,
- "variant": variant
- }
- data.update(self.get_dynamic_data(
- variant, task_name, asset_doc, project_name, host_name
- ))
-
- new_instance = CreatedInstance(
- self.family, subset_name, data, self
- )
- self._add_instance_to_context(new_instance)
-
- elif (
- existing_instance["asset"] != asset_name
- or existing_instance["task"] != task_name
- ):
- asset_doc = io.find_one({"type": "asset", "name": asset_name})
- subset_name = self.get_subset_name(
- variant, task_name, asset_doc, project_name, host_name
- )
- existing_instance["asset"] = asset_name
- existing_instance["task"] = task_name
diff --git a/openpype/hosts/testhost/plugins/create/test_creator_1.py b/openpype/hosts/testhost/plugins/create/test_creator_1.py
deleted file mode 100644
index 6ec4d164675..00000000000
--- a/openpype/hosts/testhost/plugins/create/test_creator_1.py
+++ /dev/null
@@ -1,70 +0,0 @@
-from openpype import resources
-from openpype.hosts.testhost.api import pipeline
-from openpype.pipeline import (
- Creator,
- CreatedInstance,
- lib
-)
-
-
-class TestCreatorOne(Creator):
- identifier = "test_one"
- label = "test"
- family = "test"
- description = "Testing creator of testhost"
-
- def get_icon(self):
- return resources.get_openpype_splash_filepath()
-
- def collect_instances(self):
- for instance_data in pipeline.list_instances():
- creator_id = instance_data.get("creator_identifier")
- if creator_id == self.identifier:
- instance = CreatedInstance.from_existing(
- instance_data, self
- )
- self._add_instance_to_context(instance)
-
- def update_instances(self, update_list):
- pipeline.update_instances(update_list)
-
- def remove_instances(self, instances):
- pipeline.remove_instances(instances)
- for instance in instances:
- self._remove_instance_from_context(instance)
-
- def create(self, subset_name, data, options=None):
- new_instance = CreatedInstance(self.family, subset_name, data, self)
- pipeline.HostContext.add_instance(new_instance.data_to_store())
- self.log.info(new_instance.data)
- self._add_instance_to_context(new_instance)
-
- def get_default_variants(self):
- return [
- "myVariant",
- "variantTwo",
- "different_variant"
- ]
-
- def get_attribute_defs(self):
- output = [
- lib.NumberDef("number_key", label="Number")
- ]
- return output
-
- def get_detail_description(self):
- return """# Relictus funes est Nyseides currusque nunc oblita
-
-## Causa sed
-
-Lorem markdownum posito consumptis, *plebe Amorque*, abstitimus rogatus fictaque
-gladium Circe, nos? Bos aeternum quae. Utque me, si aliquem cladis, et vestigia
-arbor, sic mea ferre lacrimae agantur prospiciens hactenus. Amanti dentes pete,
-vos quid laudemque rastrorumque terras in gratantibus **radix** erat cedemus?
-
-Pudor tu ponderibus verbaque illa; ire ergo iam Venus patris certe longae
-cruentum lecta, et quaeque. Sit doce nox. Anteit ad tempora magni plenaque et
-videres mersit sibique auctor in tendunt mittit cunctos ventisque gravitate
-volucris quemquam Aeneaden. Pectore Mensis somnus; pectora
-[ferunt](http://www.mox.org/oculosbracchia)? Fertilitatis bella dulce et suum?
- """
diff --git a/openpype/hosts/testhost/plugins/create/test_creator_2.py b/openpype/hosts/testhost/plugins/create/test_creator_2.py
deleted file mode 100644
index 4b1430a6a24..00000000000
--- a/openpype/hosts/testhost/plugins/create/test_creator_2.py
+++ /dev/null
@@ -1,74 +0,0 @@
-from openpype.hosts.testhost.api import pipeline
-from openpype.pipeline import (
- Creator,
- CreatedInstance,
- lib
-)
-
-
-class TestCreatorTwo(Creator):
- identifier = "test_two"
- label = "test"
- family = "test"
- description = "A second testing creator"
-
- def get_icon(self):
- return "cube"
-
- def create(self, subset_name, data, options=None):
- new_instance = CreatedInstance(self.family, subset_name, data, self)
- pipeline.HostContext.add_instance(new_instance.data_to_store())
- self.log.info(new_instance.data)
- self._add_instance_to_context(new_instance)
-
- def collect_instances(self):
- for instance_data in pipeline.list_instances():
- creator_id = instance_data.get("creator_identifier")
- if creator_id == self.identifier:
- instance = CreatedInstance.from_existing(
- instance_data, self
- )
- self._add_instance_to_context(instance)
-
- def update_instances(self, update_list):
- pipeline.update_instances(update_list)
-
- def remove_instances(self, instances):
- pipeline.remove_instances(instances)
- for instance in instances:
- self._remove_instance_from_context(instance)
-
- def get_attribute_defs(self):
- output = [
- lib.NumberDef("number_key"),
- lib.TextDef("text_key")
- ]
- return output
-
- def get_detail_description(self):
- return """# Lorem ipsum, dolor sit amet. [](https://github.com/sindresorhus/awesome)
-
-> A curated list of awesome lorem ipsum generators.
-
-Inspired by the [awesome](https://github.com/sindresorhus/awesome) list thing.
-
-
-## Table of Contents
-
-- [Legend](#legend)
-- [Practical](#briefcase-practical)
-- [Whimsical](#roller_coaster-whimsical)
- - [Animals](#rabbit-animals)
- - [Eras](#tophat-eras)
- - [Famous Individuals](#sunglasses-famous-individuals)
- - [Music](#microphone-music)
- - [Food and Drink](#pizza-food-and-drink)
- - [Geographic and Dialects](#earth_africa-geographic-and-dialects)
- - [Literature](#books-literature)
- - [Miscellaneous](#cyclone-miscellaneous)
- - [Sports and Fitness](#bicyclist-sports-and-fitness)
- - [TV and Film](#movie_camera-tv-and-film)
-- [Tools, Apps, and Extensions](#wrench-tools-apps-and-extensions)
-- [Contribute](#contribute)
-- [TODO](#todo)
-"""
diff --git a/openpype/hosts/testhost/plugins/publish/collect_context.py b/openpype/hosts/testhost/plugins/publish/collect_context.py
deleted file mode 100644
index 0ab98fb84b9..00000000000
--- a/openpype/hosts/testhost/plugins/publish/collect_context.py
+++ /dev/null
@@ -1,34 +0,0 @@
-import pyblish.api
-
-from openpype.pipeline import (
- OpenPypePyblishPluginMixin,
- attribute_definitions
-)
-
-
-class CollectContextDataTestHost(
- pyblish.api.ContextPlugin, OpenPypePyblishPluginMixin
-):
- """
- Collecting temp json data sent from a host context
- and path for returning json data back to hostself.
- """
-
- label = "Collect Source - Test Host"
- order = pyblish.api.CollectorOrder - 0.4
- hosts = ["testhost"]
-
- @classmethod
- def get_attribute_defs(cls):
- return [
- attribute_definitions.BoolDef(
- "test_bool",
- True,
- label="Bool input"
- )
- ]
-
- def process(self, context):
- # get json paths from os and load them
- for instance in context:
- instance.data["source"] = "testhost"
diff --git a/openpype/hosts/testhost/plugins/publish/collect_instance_1.py b/openpype/hosts/testhost/plugins/publish/collect_instance_1.py
deleted file mode 100644
index 3c035eccb6b..00000000000
--- a/openpype/hosts/testhost/plugins/publish/collect_instance_1.py
+++ /dev/null
@@ -1,54 +0,0 @@
-import json
-import pyblish.api
-
-from openpype.pipeline import (
- OpenPypePyblishPluginMixin,
- attribute_definitions
-)
-
-
-class CollectInstanceOneTestHost(
- pyblish.api.InstancePlugin, OpenPypePyblishPluginMixin
-):
- """
- Collecting temp json data sent from a host context
- and path for returning json data back to hostself.
- """
-
- label = "Collect Instance 1 - Test Host"
- order = pyblish.api.CollectorOrder - 0.3
- hosts = ["testhost"]
-
- @classmethod
- def get_attribute_defs(cls):
- return [
- attribute_definitions.NumberDef(
- "version",
- default=1,
- minimum=1,
- maximum=999,
- decimals=0,
- label="Version"
- )
- ]
-
- def process(self, instance):
- self._debug_log(instance)
-
- publish_attributes = instance.data.get("publish_attributes")
- if not publish_attributes:
- return
-
- values = publish_attributes.get(self.__class__.__name__)
- if not values:
- return
-
- instance.data["version"] = values["version"]
-
- def _debug_log(self, instance):
- def _default_json(value):
- return str(value)
-
- self.log.info(
- json.dumps(instance.data, indent=4, default=_default_json)
- )
diff --git a/openpype/hosts/testhost/plugins/publish/validate_context_with_error.py b/openpype/hosts/testhost/plugins/publish/validate_context_with_error.py
deleted file mode 100644
index 46e996a569a..00000000000
--- a/openpype/hosts/testhost/plugins/publish/validate_context_with_error.py
+++ /dev/null
@@ -1,57 +0,0 @@
-import pyblish.api
-from openpype.pipeline import PublishValidationError
-
-
-class ValidateInstanceAssetRepair(pyblish.api.Action):
- """Repair the instance asset."""
-
- label = "Repair"
- icon = "wrench"
- on = "failed"
-
- def process(self, context, plugin):
- pass
-
-
-description = """
-## Publish plugins
-
-### Validate Scene Settings
-
-#### Skip Resolution Check for Tasks
-
-Set regex pattern(s) to look for in a Task name to skip resolution check against values from DB.
-
-#### Skip Timeline Check for Tasks
-
-Set regex pattern(s) to look for in a Task name to skip `frameStart`, `frameEnd` check against values from DB.
-
-### AfterEffects Submit to Deadline
-
-* `Use Published scene` - Set to True (green) when Deadline should take published scene as a source instead of uploaded local one.
-* `Priority` - priority of job on farm
-* `Primary Pool` - here is list of pool fetched from server you can select from.
-* `Secondary Pool`
-* `Frames Per Task` - number of sequence division between individual tasks (chunks)
-making one job on farm.
-"""
-
-
-class ValidateContextWithError(pyblish.api.ContextPlugin):
- """Validate the instance asset is the current selected context asset.
-
- As it might happen that multiple worfiles are opened, switching
- between them would mess with selected context.
- In that case outputs might be output under wrong asset!
-
- Repair action will use Context asset value (from Workfiles or Launcher)
- Closing and reopening with Workfiles will refresh Context value.
- """
-
- label = "Validate Context With Error"
- hosts = ["testhost"]
- actions = [ValidateInstanceAssetRepair]
- order = pyblish.api.ValidatorOrder
-
- def process(self, context):
- raise PublishValidationError("Crashing", "Context error", description)
diff --git a/openpype/hosts/testhost/plugins/publish/validate_with_error.py b/openpype/hosts/testhost/plugins/publish/validate_with_error.py
deleted file mode 100644
index 5a2888a8b0c..00000000000
--- a/openpype/hosts/testhost/plugins/publish/validate_with_error.py
+++ /dev/null
@@ -1,57 +0,0 @@
-import pyblish.api
-from openpype.pipeline import PublishValidationError
-
-
-class ValidateInstanceAssetRepair(pyblish.api.Action):
- """Repair the instance asset."""
-
- label = "Repair"
- icon = "wrench"
- on = "failed"
-
- def process(self, context, plugin):
- pass
-
-
-description = """
-## Publish plugins
-
-### Validate Scene Settings
-
-#### Skip Resolution Check for Tasks
-
-Set regex pattern(s) to look for in a Task name to skip resolution check against values from DB.
-
-#### Skip Timeline Check for Tasks
-
-Set regex pattern(s) to look for in a Task name to skip `frameStart`, `frameEnd` check against values from DB.
-
-### AfterEffects Submit to Deadline
-
-* `Use Published scene` - Set to True (green) when Deadline should take published scene as a source instead of uploaded local one.
-* `Priority` - priority of job on farm
-* `Primary Pool` - here is list of pool fetched from server you can select from.
-* `Secondary Pool`
-* `Frames Per Task` - number of sequence division between individual tasks (chunks)
-making one job on farm.
-"""
-
-
-class ValidateWithError(pyblish.api.InstancePlugin):
- """Validate the instance asset is the current selected context asset.
-
- As it might happen that multiple worfiles are opened, switching
- between them would mess with selected context.
- In that case outputs might be output under wrong asset!
-
- Repair action will use Context asset value (from Workfiles or Launcher)
- Closing and reopening with Workfiles will refresh Context value.
- """
-
- label = "Validate With Error"
- hosts = ["testhost"]
- actions = [ValidateInstanceAssetRepair]
- order = pyblish.api.ValidatorOrder
-
- def process(self, instance):
- raise PublishValidationError("Crashing", "Instance error", description)
diff --git a/openpype/hosts/testhost/run_publish.py b/openpype/hosts/testhost/run_publish.py
deleted file mode 100644
index 44860a30e46..00000000000
--- a/openpype/hosts/testhost/run_publish.py
+++ /dev/null
@@ -1,70 +0,0 @@
-import os
-import sys
-
-mongo_url = ""
-project_name = ""
-asset_name = ""
-task_name = ""
-ftrack_url = ""
-ftrack_username = ""
-ftrack_api_key = ""
-
-
-def multi_dirname(path, times=1):
- for _ in range(times):
- path = os.path.dirname(path)
- return path
-
-
-host_name = "testhost"
-current_file = os.path.abspath(__file__)
-openpype_dir = multi_dirname(current_file, 4)
-
-os.environ["OPENPYPE_MONGO"] = mongo_url
-os.environ["OPENPYPE_ROOT"] = openpype_dir
-os.environ["AVALON_MONGO"] = mongo_url
-os.environ["AVALON_PROJECT"] = project_name
-os.environ["AVALON_ASSET"] = asset_name
-os.environ["AVALON_TASK"] = task_name
-os.environ["AVALON_APP"] = host_name
-os.environ["OPENPYPE_DATABASE_NAME"] = "openpype"
-os.environ["AVALON_CONFIG"] = "openpype"
-os.environ["AVALON_TIMEOUT"] = "1000"
-os.environ["AVALON_DB"] = "avalon"
-os.environ["FTRACK_SERVER"] = ftrack_url
-os.environ["FTRACK_API_USER"] = ftrack_username
-os.environ["FTRACK_API_KEY"] = ftrack_api_key
-for path in [
- openpype_dir,
- r"{}\repos\avalon-core".format(openpype_dir),
- r"{}\.venv\Lib\site-packages".format(openpype_dir)
-]:
- sys.path.append(path)
-
-from Qt import QtWidgets, QtCore
-
-from openpype.tools.publisher.window import PublisherWindow
-
-
-def main():
- """Main function for testing purposes."""
- import avalon.api
- import pyblish.api
- from openpype.modules import ModulesManager
- from openpype.hosts.testhost import api as testhost
-
- manager = ModulesManager()
- for plugin_path in manager.collect_plugin_paths()["publish"]:
- pyblish.api.register_plugin_path(plugin_path)
-
- avalon.api.install(testhost)
-
- QtWidgets.QApplication.setAttribute(QtCore.Qt.AA_EnableHighDpiScaling)
- app = QtWidgets.QApplication([])
- window = PublisherWindow()
- window.show()
- app.exec_()
-
-
-if __name__ == "__main__":
- main()
diff --git a/openpype/hosts/traypublisher/__init__.py b/openpype/hosts/traypublisher/__init__.py
new file mode 100644
index 00000000000..77ba908ddd2
--- /dev/null
+++ b/openpype/hosts/traypublisher/__init__.py
@@ -0,0 +1,6 @@
+from .addon import TrayPublishAddon
+
+
+__all__ = (
+ "TrayPublishAddon",
+)
diff --git a/openpype/hosts/traypublisher/addon.py b/openpype/hosts/traypublisher/addon.py
new file mode 100644
index 00000000000..c86c835ed9e
--- /dev/null
+++ b/openpype/hosts/traypublisher/addon.py
@@ -0,0 +1,67 @@
+import os
+
+import click
+
+from openpype.lib import get_openpype_execute_args
+from openpype.lib.execute import run_detached_process
+from openpype.modules import OpenPypeModule
+from openpype.modules.interfaces import ITrayAction, IHostAddon
+
+TRAYPUBLISH_ROOT_DIR = os.path.dirname(os.path.abspath(__file__))
+
+
+class TrayPublishAddon(OpenPypeModule, IHostAddon, ITrayAction):
+ label = "New Publish (beta)"
+ name = "traypublisher"
+ host_name = "traypublisher"
+
+ def initialize(self, modules_settings):
+ self.enabled = True
+ self.publish_paths = [
+ os.path.join(TRAYPUBLISH_ROOT_DIR, "plugins", "publish")
+ ]
+ self._experimental_tools = None
+
+ def tray_init(self):
+ from openpype.tools.experimental_tools import ExperimentalTools
+
+ self._experimental_tools = ExperimentalTools()
+
+ def tray_menu(self, *args, **kwargs):
+ super(TrayPublishAddon, self).tray_menu(*args, **kwargs)
+ traypublisher = self._experimental_tools.get("traypublisher")
+ visible = False
+ if traypublisher and traypublisher.enabled:
+ visible = True
+ self._action_item.setVisible(visible)
+
+ def on_action_trigger(self):
+ self.run_traypublisher()
+
+ def connect_with_modules(self, enabled_modules):
+ """Collect publish paths from other modules."""
+ publish_paths = self.manager.collect_plugin_paths()["publish"]
+ self.publish_paths.extend(publish_paths)
+
+ def run_traypublisher(self):
+ args = get_openpype_execute_args(
+ "module", self.name, "launch"
+ )
+ run_detached_process(args)
+
+ def cli(self, click_group):
+ click_group.add_command(cli_main)
+
+
+@click.group(TrayPublishAddon.name, help="TrayPublisher related commands.")
+def cli_main():
+ pass
+
+
+@cli_main.command()
+def launch():
+ """Launch TrayPublish tool UI."""
+
+ from openpype.tools import traypublisher
+
+ traypublisher.main()
diff --git a/openpype/hosts/traypublisher/api/__init__.py b/openpype/hosts/traypublisher/api/__init__.py
new file mode 100644
index 00000000000..4e7284b09a5
--- /dev/null
+++ b/openpype/hosts/traypublisher/api/__init__.py
@@ -0,0 +1,8 @@
+from .pipeline import (
+ TrayPublisherHost,
+)
+
+
+__all__ = (
+ "TrayPublisherHost",
+)
diff --git a/openpype/hosts/traypublisher/api/editorial.py b/openpype/hosts/traypublisher/api/editorial.py
new file mode 100644
index 00000000000..7c392ef5085
--- /dev/null
+++ b/openpype/hosts/traypublisher/api/editorial.py
@@ -0,0 +1,331 @@
+import re
+from copy import deepcopy
+
+from openpype.client import get_asset_by_id
+from openpype.pipeline.create import CreatorError
+
+
+class ShotMetadataSolver:
+ """ Solving hierarchical metadata
+
+ Used during editorial publishing. Works with imput
+ clip name and settings defining python formatable
+ template. Settings also define searching patterns
+ and its token keys used for formating in templates.
+ """
+
+ NO_DECOR_PATERN = re.compile(r"\{([a-z]*?)\}")
+
+ # presets
+ clip_name_tokenizer = None
+ shot_rename = True
+ shot_hierarchy = None
+ shot_add_tasks = None
+
+ def __init__(
+ self,
+ clip_name_tokenizer,
+ shot_rename,
+ shot_hierarchy,
+ shot_add_tasks,
+ logger
+ ):
+ self.clip_name_tokenizer = clip_name_tokenizer
+ self.shot_rename = shot_rename
+ self.shot_hierarchy = shot_hierarchy
+ self.shot_add_tasks = shot_add_tasks
+ self.log = logger
+
+ def _rename_template(self, data):
+ """Shot renaming function
+
+ Args:
+ data (dict): formating data
+
+ Raises:
+ CreatorError: If missing keys
+
+ Returns:
+ str: formated new name
+ """
+ shot_rename_template = self.shot_rename[
+ "shot_rename_template"]
+ try:
+ # format to new shot name
+ return shot_rename_template.format(**data)
+ except KeyError as _E:
+ raise CreatorError((
+ "Make sure all keys in settings are correct:: \n\n"
+ f"From template string {shot_rename_template} > "
+ f"`{_E}` has no equivalent in \n"
+ f"{list(data.keys())} input formating keys!"
+ ))
+
+ def _generate_tokens(self, clip_name, source_data):
+ """Token generator
+
+ Settings defines token pairs key and regex expression.
+
+ Args:
+ clip_name (str): name of clip in editorial
+ source_data (dict): data for formating
+
+ Raises:
+ CreatorError: if missing key
+
+ Returns:
+ dict: updated source_data
+ """
+ output_data = deepcopy(source_data["anatomy_data"])
+ output_data["clip_name"] = clip_name
+
+ if not self.clip_name_tokenizer:
+ return output_data
+
+ parent_name = source_data["selected_asset_doc"]["name"]
+
+ search_text = parent_name + clip_name
+
+ for token_key, pattern in self.clip_name_tokenizer.items():
+ p = re.compile(pattern)
+ match = p.findall(search_text)
+ if not match:
+ raise CreatorError((
+ "Make sure regex expression works with your data: \n\n"
+ f"'{token_key}' with regex '{pattern}' in your settings\n"
+ "can't find any match in your clip name "
+ f"'{search_text}'!\n\nLook to: "
+ "'project_settings/traypublisher/editorial_creators"
+ "/editorial_simple/clip_name_tokenizer'\n"
+ "at your project settings..."
+ ))
+
+ # QUESTION:how to refactory `match[-1]` to some better way?
+ output_data[token_key] = match[-1]
+
+ return output_data
+
+ def _create_parents_from_settings(self, parents, data):
+ """Formating parent components.
+
+ Args:
+ parents (list): list of dict parent components
+ data (dict): formating data
+
+ Raises:
+ CreatorError: missing formating key
+ CreatorError: missing token key
+ KeyError: missing parent token
+
+ Returns:
+ list: list of dict of parent components
+ """
+ # fill the parents parts from presets
+ shot_hierarchy = deepcopy(self.shot_hierarchy)
+ hierarchy_parents = shot_hierarchy["parents"]
+
+ # fill parent keys data template from anatomy data
+ try:
+ _parent_tokens_formating_data = {
+ parent_token["name"]: parent_token["value"].format(**data)
+ for parent_token in hierarchy_parents
+ }
+ except KeyError as _E:
+ raise CreatorError((
+ "Make sure all keys in settings are correct : \n"
+ f"`{_E}` has no equivalent in \n{list(data.keys())}"
+ ))
+
+ _parent_tokens_type = {
+ parent_token["name"]: parent_token["type"]
+ for parent_token in hierarchy_parents
+ }
+ for _index, _parent in enumerate(
+ shot_hierarchy["parents_path"].split("/")
+ ):
+ # format parent token with value which is formated
+ try:
+ parent_name = _parent.format(
+ **_parent_tokens_formating_data)
+ except KeyError as _E:
+ raise CreatorError((
+ "Make sure all keys in settings are correct : \n\n"
+ f"`{_E}` from template string "
+ f"{shot_hierarchy['parents_path']}, "
+ f" has no equivalent in \n"
+ f"{list(_parent_tokens_formating_data.keys())} parents"
+ ))
+
+ parent_token_name = (
+ self.NO_DECOR_PATERN.findall(_parent).pop())
+
+ if not parent_token_name:
+ raise KeyError(
+ f"Parent token is not found in: `{_parent}`")
+
+ # find parent type
+ parent_token_type = _parent_tokens_type[parent_token_name]
+
+ # in case selected context is set to the same asset
+ if (
+ _index == 0
+ and parents[-1]["entity_name"] == parent_name
+ ):
+ self.log.debug(f" skipping : {parent_name}")
+ continue
+
+ # in case first parent is project then start parents from start
+ if (
+ _index == 0
+ and parent_token_type == "Project"
+ ):
+ self.log.debug("rebuilding parents from scratch")
+ project_parent = parents[0]
+ parents = [project_parent]
+ continue
+
+ parents.append({
+ "entity_type": parent_token_type,
+ "entity_name": parent_name
+ })
+
+ self.log.debug(f"__ parents: {parents}")
+
+ return parents
+
+ def _create_hierarchy_path(self, parents):
+ """Converting hierarchy path from parents
+
+ Args:
+ parents (list): list of dict parent components
+
+ Returns:
+ str: hierarchy path
+ """
+ return "/".join(
+ [
+ p["entity_name"] for p in parents
+ if p["entity_type"] != "Project"
+ ]
+ ) if parents else ""
+
+ def _get_parents_from_selected_asset(
+ self,
+ asset_doc,
+ project_doc
+ ):
+ """Returning parents from context on selected asset.
+
+ Context defined in Traypublisher project tree.
+
+ Args:
+ asset_doc (db obj): selected asset doc
+ project_doc (db obj): actual project doc
+
+ Returns:
+ list: list of dict parent components
+ """
+ project_name = project_doc["name"]
+ visual_hierarchy = [asset_doc]
+ current_doc = asset_doc
+
+ # looping trought all available visual parents
+ # if they are not available anymore than it breaks
+ while True:
+ visual_parent_id = current_doc["data"]["visualParent"]
+ visual_parent = None
+ if visual_parent_id:
+ visual_parent = get_asset_by_id(project_name, visual_parent_id)
+
+ if not visual_parent:
+ visual_hierarchy.append(project_doc)
+ break
+ visual_hierarchy.append(visual_parent)
+ current_doc = visual_parent
+
+ # add current selection context hierarchy
+ return [
+ {
+ "entity_type": entity["data"]["entityType"],
+ "entity_name": entity["name"]
+ }
+ for entity in reversed(visual_hierarchy)
+ ]
+
+ def _generate_tasks_from_settings(self, project_doc):
+ """Convert settings inputs to task data.
+
+ Args:
+ project_doc (db obj): actual project doc
+
+ Raises:
+ KeyError: Missing task type in project doc
+
+ Returns:
+ dict: tasks data
+ """
+ tasks_to_add = {}
+
+ project_tasks = project_doc["config"]["tasks"]
+ for task_name, task_data in self.shot_add_tasks.items():
+ _task_data = deepcopy(task_data)
+
+ # check if task type in project task types
+ if _task_data["type"] in project_tasks.keys():
+ tasks_to_add[task_name] = _task_data
+ else:
+ raise KeyError(
+ "Missing task type `{}` for `{}` is not"
+ " existing in `{}``".format(
+ _task_data["type"],
+ task_name,
+ list(project_tasks.keys())
+ )
+ )
+
+ return tasks_to_add
+
+ def generate_data(self, clip_name, source_data):
+ """Metadata generator.
+
+ Converts input data to hierarchy mentadata.
+
+ Args:
+ clip_name (str): clip name
+ source_data (dict): formating data
+
+ Returns:
+ (str, dict): shot name and hierarchy data
+ """
+ self.log.info(f"_ source_data: {source_data}")
+
+ tasks = {}
+ asset_doc = source_data["selected_asset_doc"]
+ project_doc = source_data["project_doc"]
+
+ # match clip to shot name at start
+ shot_name = clip_name
+
+ # parse all tokens and generate formating data
+ formating_data = self._generate_tokens(shot_name, source_data)
+
+ # generate parents from selected asset
+ parents = self._get_parents_from_selected_asset(asset_doc, project_doc)
+
+ if self.shot_rename["enabled"]:
+ shot_name = self._rename_template(formating_data)
+ self.log.info(f"Renamed shot name: {shot_name}")
+
+ if self.shot_hierarchy["enabled"]:
+ parents = self._create_parents_from_settings(
+ parents, formating_data)
+
+ if self.shot_add_tasks:
+ tasks = self._generate_tasks_from_settings(
+ project_doc)
+
+ return shot_name, {
+ "hierarchy": self._create_hierarchy_path(parents),
+ "parents": parents,
+ "tasks": tasks
+ }
diff --git a/openpype/hosts/traypublisher/api/pipeline.py b/openpype/hosts/traypublisher/api/pipeline.py
new file mode 100644
index 00000000000..2d9db7801ea
--- /dev/null
+++ b/openpype/hosts/traypublisher/api/pipeline.py
@@ -0,0 +1,183 @@
+import os
+import json
+import tempfile
+import atexit
+
+import pyblish.api
+
+from openpype.pipeline import (
+ register_creator_plugin_path,
+ legacy_io,
+)
+from openpype.host import HostBase, INewPublisher
+
+
+ROOT_DIR = os.path.dirname(os.path.dirname(
+ os.path.abspath(__file__)
+))
+PUBLISH_PATH = os.path.join(ROOT_DIR, "plugins", "publish")
+CREATE_PATH = os.path.join(ROOT_DIR, "plugins", "create")
+
+
+class TrayPublisherHost(HostBase, INewPublisher):
+ name = "traypublisher"
+
+ def install(self):
+ os.environ["AVALON_APP"] = self.name
+ legacy_io.Session["AVALON_APP"] = self.name
+
+ pyblish.api.register_host("traypublisher")
+ pyblish.api.register_plugin_path(PUBLISH_PATH)
+ register_creator_plugin_path(CREATE_PATH)
+
+ def get_context_title(self):
+ return HostContext.get_project_name()
+
+ def get_context_data(self):
+ return HostContext.get_context_data()
+
+ def update_context_data(self, data, changes):
+ HostContext.save_context_data(data, changes)
+
+ def set_project_name(self, project_name):
+ # TODO Deregister project specific plugins and register new project
+ # plugins
+ os.environ["AVALON_PROJECT"] = project_name
+ legacy_io.Session["AVALON_PROJECT"] = project_name
+ legacy_io.install()
+ HostContext.set_project_name(project_name)
+
+
+class HostContext:
+ _context_json_path = None
+
+ @staticmethod
+ def _on_exit():
+ if (
+ HostContext._context_json_path
+ and os.path.exists(HostContext._context_json_path)
+ ):
+ os.remove(HostContext._context_json_path)
+
+ @classmethod
+ def get_context_json_path(cls):
+ if cls._context_json_path is None:
+ output_file = tempfile.NamedTemporaryFile(
+ mode="w", prefix="traypub_", suffix=".json"
+ )
+ output_file.close()
+ cls._context_json_path = output_file.name
+ atexit.register(HostContext._on_exit)
+ print(cls._context_json_path)
+ return cls._context_json_path
+
+ @classmethod
+ def _get_data(cls, group=None):
+ json_path = cls.get_context_json_path()
+ data = {}
+ if not os.path.exists(json_path):
+ with open(json_path, "w") as json_stream:
+ json.dump(data, json_stream)
+ else:
+ with open(json_path, "r") as json_stream:
+ content = json_stream.read()
+ if content:
+ data = json.loads(content)
+ if group is None:
+ return data
+ return data.get(group)
+
+ @classmethod
+ def _save_data(cls, group, new_data):
+ json_path = cls.get_context_json_path()
+ data = cls._get_data()
+ data[group] = new_data
+ with open(json_path, "w") as json_stream:
+ json.dump(data, json_stream)
+
+ @classmethod
+ def add_instance(cls, instance):
+ instances = cls.get_instances()
+ instances.append(instance)
+ cls.save_instances(instances)
+
+ @classmethod
+ def get_instances(cls):
+ return cls._get_data("instances") or []
+
+ @classmethod
+ def save_instances(cls, instances):
+ cls._save_data("instances", instances)
+
+ @classmethod
+ def get_context_data(cls):
+ return cls._get_data("context") or {}
+
+ @classmethod
+ def save_context_data(cls, data):
+ cls._save_data("context", data)
+
+ @classmethod
+ def get_project_name(cls):
+ return cls._get_data("project_name")
+
+ @classmethod
+ def set_project_name(cls, project_name):
+ cls._save_data("project_name", project_name)
+
+ @classmethod
+ def get_data_to_store(cls):
+ return {
+ "project_name": cls.get_project_name(),
+ "instances": cls.get_instances(),
+ "context": cls.get_context_data(),
+ }
+
+
+def list_instances():
+ return HostContext.get_instances()
+
+
+def update_instances(update_list):
+ updated_instances = {}
+ for instance, _changes in update_list:
+ updated_instances[instance.id] = instance.data_to_store()
+
+ instances = HostContext.get_instances()
+ for instance_data in instances:
+ instance_id = instance_data["instance_id"]
+ if instance_id in updated_instances:
+ new_instance_data = updated_instances[instance_id]
+ old_keys = set(instance_data.keys())
+ new_keys = set(new_instance_data.keys())
+ instance_data.update(new_instance_data)
+ for key in (old_keys - new_keys):
+ instance_data.pop(key)
+
+ HostContext.save_instances(instances)
+
+
+def remove_instances(instances):
+ if not isinstance(instances, (tuple, list)):
+ instances = [instances]
+
+ current_instances = HostContext.get_instances()
+ for instance in instances:
+ instance_id = instance.data["instance_id"]
+ found_idx = None
+ for idx, _instance in enumerate(current_instances):
+ if instance_id == _instance["instance_id"]:
+ found_idx = idx
+ break
+
+ if found_idx is not None:
+ current_instances.pop(found_idx)
+ HostContext.save_instances(current_instances)
+
+
+def get_context_data():
+ return HostContext.get_context_data()
+
+
+def update_context_data(data, changes):
+ HostContext.save_context_data(data)
diff --git a/openpype/hosts/traypublisher/api/plugin.py b/openpype/hosts/traypublisher/api/plugin.py
new file mode 100644
index 00000000000..a3eead51c82
--- /dev/null
+++ b/openpype/hosts/traypublisher/api/plugin.py
@@ -0,0 +1,171 @@
+from openpype.lib.attribute_definitions import FileDef
+from openpype.pipeline.create import (
+ Creator,
+ HiddenCreator,
+ CreatedInstance
+)
+
+from .pipeline import (
+ list_instances,
+ update_instances,
+ remove_instances,
+ HostContext,
+)
+
+IMAGE_EXTENSIONS = [
+ ".ani", ".anim", ".apng", ".art", ".bmp", ".bpg", ".bsave", ".cal",
+ ".cin", ".cpc", ".cpt", ".dds", ".dpx", ".ecw", ".exr", ".fits",
+ ".flic", ".flif", ".fpx", ".gif", ".hdri", ".hevc", ".icer",
+ ".icns", ".ico", ".cur", ".ics", ".ilbm", ".jbig", ".jbig2",
+ ".jng", ".jpeg", ".jpeg-ls", ".jpeg", ".2000", ".jpg", ".xr",
+ ".jpeg", ".xt", ".jpeg-hdr", ".kra", ".mng", ".miff", ".nrrd",
+ ".ora", ".pam", ".pbm", ".pgm", ".ppm", ".pnm", ".pcx", ".pgf",
+ ".pictor", ".png", ".psb", ".psp", ".qtvr", ".ras",
+ ".rgbe", ".logluv", ".tiff", ".sgi", ".tga", ".tiff", ".tiff/ep",
+ ".tiff/it", ".ufo", ".ufp", ".wbmp", ".webp", ".xbm", ".xcf",
+ ".xpm", ".xwd"
+]
+VIDEO_EXTENSIONS = [
+ ".3g2", ".3gp", ".amv", ".asf", ".avi", ".drc", ".f4a", ".f4b",
+ ".f4p", ".f4v", ".flv", ".gif", ".gifv", ".m2v", ".m4p", ".m4v",
+ ".mkv", ".mng", ".mov", ".mp2", ".mp4", ".mpe", ".mpeg", ".mpg",
+ ".mpv", ".mxf", ".nsv", ".ogg", ".ogv", ".qt", ".rm", ".rmvb",
+ ".roq", ".svi", ".vob", ".webm", ".wmv", ".yuv"
+]
+REVIEW_EXTENSIONS = IMAGE_EXTENSIONS + VIDEO_EXTENSIONS
+
+
+class HiddenTrayPublishCreator(HiddenCreator):
+ host_name = "traypublisher"
+
+ def collect_instances(self):
+ for instance_data in list_instances():
+ creator_id = instance_data.get("creator_identifier")
+ if creator_id == self.identifier:
+ instance = CreatedInstance.from_existing(
+ instance_data, self
+ )
+ self._add_instance_to_context(instance)
+
+ def update_instances(self, update_list):
+ update_instances(update_list)
+
+ def remove_instances(self, instances):
+ remove_instances(instances)
+ for instance in instances:
+ self._remove_instance_from_context(instance)
+
+ def _store_new_instance(self, new_instance):
+ """Tray publisher specific method to store instance.
+
+ Instance is stored into "workfile" of traypublisher and also add it
+ to CreateContext.
+
+ Args:
+ new_instance (CreatedInstance): Instance that should be stored.
+ """
+
+ # Host implementation of storing metadata about instance
+ HostContext.add_instance(new_instance.data_to_store())
+ # Add instance to current context
+ self._add_instance_to_context(new_instance)
+
+
+class TrayPublishCreator(Creator):
+ create_allow_context_change = True
+ host_name = "traypublisher"
+
+ def collect_instances(self):
+ for instance_data in list_instances():
+ creator_id = instance_data.get("creator_identifier")
+ if creator_id == self.identifier:
+ instance = CreatedInstance.from_existing(
+ instance_data, self
+ )
+ self._add_instance_to_context(instance)
+
+ def update_instances(self, update_list):
+ update_instances(update_list)
+
+ def remove_instances(self, instances):
+ remove_instances(instances)
+ for instance in instances:
+ self._remove_instance_from_context(instance)
+
+ def _store_new_instance(self, new_instance):
+ """Tray publisher specific method to store instance.
+
+ Instance is stored into "workfile" of traypublisher and also add it
+ to CreateContext.
+
+ Args:
+ new_instance (CreatedInstance): Instance that should be stored.
+ """
+
+ # Host implementation of storing metadata about instance
+ HostContext.add_instance(new_instance.data_to_store())
+ # Add instance to current context
+ self._add_instance_to_context(new_instance)
+
+
+class SettingsCreator(TrayPublishCreator):
+ create_allow_context_change = True
+
+ extensions = []
+
+ def create(self, subset_name, data, pre_create_data):
+ # Pass precreate data to creator attributes
+ data["creator_attributes"] = pre_create_data
+ data["settings_creator"] = True
+ # Create new instance
+ new_instance = CreatedInstance(self.family, subset_name, data, self)
+
+ self._store_new_instance(new_instance)
+
+ def get_instance_attr_defs(self):
+ return [
+ FileDef(
+ "representation_files",
+ folders=False,
+ extensions=self.extensions,
+ allow_sequences=self.allow_sequences,
+ single_item=not self.allow_multiple_items,
+ label="Representations",
+ ),
+ FileDef(
+ "reviewable",
+ folders=False,
+ extensions=REVIEW_EXTENSIONS,
+ allow_sequences=True,
+ single_item=True,
+ label="Reviewable representations",
+ extensions_label="Single reviewable item"
+ )
+ ]
+
+ def get_pre_create_attr_defs(self):
+ # Use same attributes as for instance attrobites
+ return self.get_instance_attr_defs()
+
+ @classmethod
+ def from_settings(cls, item_data):
+ identifier = item_data["identifier"]
+ family = item_data["family"]
+ if not identifier:
+ identifier = "settings_{}".format(family)
+ return type(
+ "{}{}".format(cls.__name__, identifier),
+ (cls, ),
+ {
+ "family": family,
+ "identifier": identifier,
+ "label": item_data["label"].strip(),
+ "icon": item_data["icon"],
+ "description": item_data["description"],
+ "detailed_description": item_data["detailed_description"],
+ "extensions": item_data["extensions"],
+ "allow_sequences": item_data["allow_sequences"],
+ "allow_multiple_items": item_data["allow_multiple_items"],
+ "default_variants": item_data["default_variants"]
+ }
+ )
diff --git a/openpype/hosts/traypublisher/plugins/create/create_editorial.py b/openpype/hosts/traypublisher/plugins/create/create_editorial.py
new file mode 100644
index 00000000000..28a115629ed
--- /dev/null
+++ b/openpype/hosts/traypublisher/plugins/create/create_editorial.py
@@ -0,0 +1,869 @@
+import os
+from copy import deepcopy
+from pprint import pformat
+import opentimelineio as otio
+from openpype.client import (
+ get_asset_by_name,
+ get_project
+)
+from openpype.hosts.traypublisher.api.plugin import (
+ TrayPublishCreator,
+ HiddenTrayPublishCreator
+)
+from openpype.hosts.traypublisher.api.editorial import (
+ ShotMetadataSolver
+)
+
+from openpype.pipeline import CreatedInstance
+
+from openpype.lib import (
+ get_ffprobe_data,
+ convert_ffprobe_fps_value,
+
+ FileDef,
+ TextDef,
+ NumberDef,
+ EnumDef,
+ BoolDef,
+ UISeparatorDef,
+ UILabelDef
+)
+
+
+CLIP_ATTR_DEFS = [
+ EnumDef(
+ "fps",
+ items={
+ "from_selection": "From selection",
+ 23.997: "23.976",
+ 24: "24",
+ 25: "25",
+ 29.97: "29.97",
+ 30: "30"
+ },
+ label="FPS"
+ ),
+ NumberDef(
+ "workfile_start_frame",
+ default=1001,
+ label="Workfile start frame"
+ ),
+ NumberDef(
+ "handle_start",
+ default=0,
+ label="Handle start"
+ ),
+ NumberDef(
+ "handle_end",
+ default=0,
+ label="Handle end"
+ )
+]
+
+
+class EditorialClipInstanceCreatorBase(HiddenTrayPublishCreator):
+ """ Wrapper class for clip family creators
+
+ Args:
+ HiddenTrayPublishCreator (BaseCreator): hidden supporting class
+ """
+ host_name = "traypublisher"
+
+ def create(self, instance_data, source_data=None):
+ self.log.info(f"instance_data: {instance_data}")
+ subset_name = instance_data["subset"]
+
+ # Create new instance
+ new_instance = CreatedInstance(
+ self.family, subset_name, instance_data, self
+ )
+ self.log.info(f"instance_data: {pformat(new_instance.data)}")
+
+ self._store_new_instance(new_instance)
+
+ return new_instance
+
+ def get_instance_attr_defs(self):
+ return [
+ BoolDef(
+ "add_review_family",
+ default=True,
+ label="Review"
+ )
+ ]
+
+
+class EditorialShotInstanceCreator(EditorialClipInstanceCreatorBase):
+ """ Shot family class
+
+ The shot metadata instance carrier.
+
+ Args:
+ EditorialClipInstanceCreatorBase (BaseCreator): hidden supporting class
+ """
+ identifier = "editorial_shot"
+ family = "shot"
+ label = "Editorial Shot"
+
+ def get_instance_attr_defs(self):
+ attr_defs = [
+ TextDef(
+ "asset_name",
+ label="Asset name",
+ )
+ ]
+ attr_defs.extend(CLIP_ATTR_DEFS)
+ return attr_defs
+
+
+class EditorialPlateInstanceCreator(EditorialClipInstanceCreatorBase):
+ """ Plate family class
+
+ Plate representation instance.
+
+ Args:
+ EditorialClipInstanceCreatorBase (BaseCreator): hidden supporting class
+ """
+ identifier = "editorial_plate"
+ family = "plate"
+ label = "Editorial Plate"
+
+
+class EditorialAudioInstanceCreator(EditorialClipInstanceCreatorBase):
+ """ Audio family class
+
+ Audio representation instance.
+
+ Args:
+ EditorialClipInstanceCreatorBase (BaseCreator): hidden supporting class
+ """
+ identifier = "editorial_audio"
+ family = "audio"
+ label = "Editorial Audio"
+
+
+class EditorialReviewInstanceCreator(EditorialClipInstanceCreatorBase):
+ """ Review family class
+
+ Review representation instance.
+
+ Args:
+ EditorialClipInstanceCreatorBase (BaseCreator): hidden supporting class
+ """
+ identifier = "editorial_review"
+ family = "review"
+ label = "Editorial Review"
+
+
+class EditorialSimpleCreator(TrayPublishCreator):
+ """ Editorial creator class
+
+ Simple workflow creator. This creator only disecting input
+ video file into clip chunks and then converts each to
+ defined format defined Settings for each subset preset.
+
+ Args:
+ TrayPublishCreator (Creator): Tray publisher plugin class
+ """
+
+ label = "Editorial Simple"
+ family = "editorial"
+ identifier = "editorial_simple"
+ default_variants = [
+ "main"
+ ]
+ description = "Editorial files to generate shots."
+ detailed_description = """
+Supporting publishing new shots to project
+or updating already created. Publishing will create OTIO file.
+"""
+ icon = "fa.file"
+
+ def __init__(
+ self, project_settings, *args, **kwargs
+ ):
+ super(EditorialSimpleCreator, self).__init__(
+ project_settings, *args, **kwargs
+ )
+ editorial_creators = deepcopy(
+ project_settings["traypublisher"]["editorial_creators"]
+ )
+ # get this creator settings by identifier
+ self._creator_settings = editorial_creators.get(self.identifier)
+
+ clip_name_tokenizer = self._creator_settings["clip_name_tokenizer"]
+ shot_rename = self._creator_settings["shot_rename"]
+ shot_hierarchy = self._creator_settings["shot_hierarchy"]
+ shot_add_tasks = self._creator_settings["shot_add_tasks"]
+
+ self._shot_metadata_solver = ShotMetadataSolver(
+ clip_name_tokenizer,
+ shot_rename,
+ shot_hierarchy,
+ shot_add_tasks,
+ self.log
+ )
+
+ # try to set main attributes from settings
+ if self._creator_settings.get("default_variants"):
+ self.default_variants = self._creator_settings["default_variants"]
+
+ def create(self, subset_name, instance_data, pre_create_data):
+ allowed_family_presets = self._get_allowed_family_presets(
+ pre_create_data)
+
+ clip_instance_properties = {
+ k: v for k, v in pre_create_data.items()
+ if k != "sequence_filepath_data"
+ if k not in [
+ i["family"] for i in self._creator_settings["family_presets"]
+ ]
+ }
+ # Create otio editorial instance
+ asset_name = instance_data["asset"]
+ asset_doc = get_asset_by_name(self.project_name, asset_name)
+
+ self.log.info(pre_create_data["fps"])
+
+ if pre_create_data["fps"] == "from_selection":
+ # get asset doc data attributes
+ fps = asset_doc["data"]["fps"]
+ else:
+ fps = float(pre_create_data["fps"])
+
+ instance_data.update({
+ "fps": fps
+ })
+
+ # get path of sequence
+ sequence_path_data = pre_create_data["sequence_filepath_data"]
+ media_path_data = pre_create_data["media_filepaths_data"]
+
+ sequence_path = self._get_path_from_file_data(sequence_path_data)
+ media_path = self._get_path_from_file_data(media_path_data)
+
+ # get otio timeline
+ otio_timeline = self._create_otio_timeline(
+ sequence_path, fps)
+
+ # Create all clip instances
+ clip_instance_properties.update({
+ "fps": fps,
+ "parent_asset_name": asset_name,
+ "variant": instance_data["variant"]
+ })
+
+ # create clip instances
+ self._get_clip_instances(
+ otio_timeline,
+ media_path,
+ clip_instance_properties,
+ family_presets=allowed_family_presets
+
+ )
+
+ # create otio editorial instance
+ self._create_otio_instance(
+ subset_name, instance_data,
+ sequence_path, media_path,
+ otio_timeline
+ )
+
+ def _create_otio_instance(
+ self,
+ subset_name,
+ data,
+ sequence_path,
+ media_path,
+ otio_timeline
+ ):
+ """Otio instance creating function
+
+ Args:
+ subset_name (str): name of subset
+ data (dict): instnance data
+ sequence_path (str): path to sequence file
+ media_path (str): path to media file
+ otio_timeline (otio.Timeline): otio timeline object
+ """
+ # Pass precreate data to creator attributes
+ data.update({
+ "sequenceFilePath": sequence_path,
+ "editorialSourcePath": media_path,
+ "otioTimeline": otio.adapters.write_to_string(otio_timeline)
+ })
+ new_instance = CreatedInstance(
+ self.family, subset_name, data, self
+ )
+ self._store_new_instance(new_instance)
+
+ def _create_otio_timeline(self, sequence_path, fps):
+ """Creating otio timeline from sequence path
+
+ Args:
+ sequence_path (str): path to sequence file
+ fps (float): frame per second
+
+ Returns:
+ otio.Timeline: otio timeline object
+ """
+ # get editorial sequence file into otio timeline object
+ extension = os.path.splitext(sequence_path)[1]
+
+ kwargs = {}
+ if extension == ".edl":
+ # EDL has no frame rate embedded so needs explicit
+ # frame rate else 24 is asssumed.
+ kwargs["rate"] = fps
+ kwargs["ignore_timecode_mismatch"] = True
+
+ self.log.info(f"kwargs: {kwargs}")
+ return otio.adapters.read_from_file(sequence_path, **kwargs)
+
+ def _get_path_from_file_data(self, file_path_data):
+ """Converting creator path data to single path string
+
+ Args:
+ file_path_data (FileDefItem): creator path data inputs
+
+ Raises:
+ FileExistsError: in case nothing had been set
+
+ Returns:
+ str: path string
+ """
+ # TODO: just temporarly solving only one media file
+ if isinstance(file_path_data, list):
+ file_path_data = file_path_data.pop()
+
+ if len(file_path_data["filenames"]) == 0:
+ raise FileExistsError(
+ f"File path was not added: {file_path_data}")
+
+ return os.path.join(
+ file_path_data["directory"], file_path_data["filenames"][0])
+
+ def _get_clip_instances(
+ self,
+ otio_timeline,
+ media_path,
+ instance_data,
+ family_presets
+ ):
+ """Helping function fro creating clip instance
+
+ Args:
+ otio_timeline (otio.Timeline): otio timeline object
+ media_path (str): media file path string
+ instance_data (dict): clip instance data
+ family_presets (list): list of dict settings subset presets
+ """
+ self.asset_name_check = []
+
+ tracks = otio_timeline.each_child(
+ descended_from_type=otio.schema.Track
+ )
+
+ # media data for audio sream and reference solving
+ media_data = self._get_media_source_metadata(media_path)
+
+ for track in tracks:
+ self.log.debug(f"track.name: {track.name}")
+ try:
+ track_start_frame = (
+ abs(track.source_range.start_time.value)
+ )
+ self.log.debug(f"track_start_frame: {track_start_frame}")
+ track_start_frame -= self.timeline_frame_start
+ except AttributeError:
+ track_start_frame = 0
+
+ self.log.debug(f"track_start_frame: {track_start_frame}")
+
+ for clip in track.each_child():
+ if not self._validate_clip_for_processing(clip):
+ continue
+
+ # get available frames info to clip data
+ self._create_otio_reference(clip, media_path, media_data)
+
+ # convert timeline range to source range
+ self._restore_otio_source_range(clip)
+
+ base_instance_data = self._get_base_instance_data(
+ clip,
+ instance_data,
+ track_start_frame
+ )
+
+ parenting_data = {
+ "instance_label": None,
+ "instance_id": None
+ }
+ self.log.info((
+ "Creating subsets from presets: \n"
+ f"{pformat(family_presets)}"
+ ))
+
+ for _fpreset in family_presets:
+ # exclude audio family if no audio stream
+ if (
+ _fpreset["family"] == "audio"
+ and not media_data.get("audio")
+ ):
+ continue
+
+ instance = self._make_subset_instance(
+ clip,
+ _fpreset,
+ deepcopy(base_instance_data),
+ parenting_data
+ )
+ self.log.debug(f"{pformat(dict(instance.data))}")
+
+ def _restore_otio_source_range(self, otio_clip):
+ """Infusing source range.
+
+ Otio clip is missing proper source clip range so
+ here we add them from from parent timeline frame range.
+
+ Args:
+ otio_clip (otio.Clip): otio clip object
+ """
+ otio_clip.source_range = otio_clip.range_in_parent()
+
+ def _create_otio_reference(
+ self,
+ otio_clip,
+ media_path,
+ media_data
+ ):
+ """Creating otio reference at otio clip.
+
+ Args:
+ otio_clip (otio.Clip): otio clip object
+ media_path (str): media file path string
+ media_data (dict): media metadata
+ """
+ start_frame = media_data["start_frame"]
+ frame_duration = media_data["duration"]
+ fps = media_data["fps"]
+
+ available_range = otio.opentime.TimeRange(
+ start_time=otio.opentime.RationalTime(
+ start_frame, fps),
+ duration=otio.opentime.RationalTime(
+ frame_duration, fps)
+ )
+ # in case old OTIO or video file create `ExternalReference`
+ media_reference = otio.schema.ExternalReference(
+ target_url=media_path,
+ available_range=available_range
+ )
+
+ otio_clip.media_reference = media_reference
+
+ def _get_media_source_metadata(self, path):
+ """Get all available metadata from file
+
+ Args:
+ path (str): media file path string
+
+ Raises:
+ AssertionError: ffprobe couldn't read metadata
+
+ Returns:
+ dict: media file metadata
+ """
+ return_data = {}
+
+ try:
+ media_data = get_ffprobe_data(
+ path, self.log
+ )
+ self.log.debug(f"__ media_data: {pformat(media_data)}")
+
+ # get video stream data
+ video_stream = media_data["streams"][0]
+ return_data = {
+ "video": True,
+ "start_frame": 0,
+ "duration": int(video_stream["nb_frames"]),
+ "fps": float(
+ convert_ffprobe_fps_value(
+ video_stream["r_frame_rate"]
+ )
+ )
+ }
+
+ # get audio streams data
+ audio_stream = [
+ stream for stream in media_data["streams"]
+ if stream["codec_type"] == "audio"
+ ]
+
+ if audio_stream:
+ return_data["audio"] = True
+
+ except Exception as exc:
+ raise AssertionError((
+ "FFprobe couldn't read information about input file: "
+ f"\"{path}\". Error message: {exc}"
+ ))
+
+ return return_data
+
+ def _make_subset_instance(
+ self,
+ otio_clip,
+ preset,
+ instance_data,
+ parenting_data
+ ):
+ """Making subset instance from input preset
+
+ Args:
+ otio_clip (otio.Clip): otio clip object
+ preset (dict): sigle family preset
+ instance_data (dict): instance data
+ parenting_data (dict): shot instance parent data
+
+ Returns:
+ CreatedInstance: creator instance object
+ """
+ family = preset["family"]
+ label = self._make_subset_naming(
+ preset,
+ instance_data
+ )
+ instance_data["label"] = label
+
+ # add file extension filter only if it is not shot family
+ if family == "shot":
+ instance_data["otioClip"] = (
+ otio.adapters.write_to_string(otio_clip))
+ c_instance = self.create_context.creators[
+ "editorial_shot"].create(
+ instance_data)
+ parenting_data.update({
+ "instance_label": label,
+ "instance_id": c_instance.data["instance_id"]
+ })
+ else:
+ # add review family if defined
+ instance_data.update({
+ "outputFileType": preset["output_file_type"],
+ "parent_instance_id": parenting_data["instance_id"],
+ "creator_attributes": {
+ "parent_instance": parenting_data["instance_label"],
+ "add_review_family": preset.get("review")
+ }
+ })
+
+ creator_identifier = f"editorial_{family}"
+ editorial_clip_creator = self.create_context.creators[
+ creator_identifier]
+ c_instance = editorial_clip_creator.create(
+ instance_data)
+
+ return c_instance
+
+ def _make_subset_naming(
+ self,
+ preset,
+ instance_data
+ ):
+ """ Subset name maker
+
+ Args:
+ preset (dict): single preset item
+ instance_data (dict): instance data
+
+ Returns:
+ str: label string
+ """
+ shot_name = instance_data["shotName"]
+ variant_name = instance_data["variant"]
+ family = preset["family"]
+
+ # get variant name from preset or from inharitance
+ _variant_name = preset.get("variant") or variant_name
+
+ self.log.debug(f"__ family: {family}")
+ self.log.debug(f"__ preset: {preset}")
+
+ # subset name
+ subset_name = "{}{}".format(
+ family, _variant_name.capitalize()
+ )
+ label = "{}_{}".format(
+ shot_name,
+ subset_name
+ )
+
+ instance_data.update({
+ "family": family,
+ "label": label,
+ "variant": _variant_name,
+ "subset": subset_name,
+ })
+
+ return label
+
+ def _get_base_instance_data(
+ self,
+ otio_clip,
+ instance_data,
+ track_start_frame,
+ ):
+ """ Factoring basic set of instance data.
+
+ Args:
+ otio_clip (otio.Clip): otio clip object
+ instance_data (dict): precreate instance data
+ track_start_frame (int): track start frame
+
+ Returns:
+ dict: instance data
+ """
+ # get clip instance properties
+ parent_asset_name = instance_data["parent_asset_name"]
+ handle_start = instance_data["handle_start"]
+ handle_end = instance_data["handle_end"]
+ timeline_offset = instance_data["timeline_offset"]
+ workfile_start_frame = instance_data["workfile_start_frame"]
+ fps = instance_data["fps"]
+ variant_name = instance_data["variant"]
+
+ # basic unique asset name
+ clip_name = os.path.splitext(otio_clip.name)[0].lower()
+ project_doc = get_project(self.project_name)
+
+ shot_name, shot_metadata = self._shot_metadata_solver.generate_data(
+ clip_name,
+ {
+ "anatomy_data": {
+ "project": {
+ "name": self.project_name,
+ "code": project_doc["data"]["code"]
+ },
+ "parent": parent_asset_name,
+ "app": self.host_name
+ },
+ "selected_asset_doc": get_asset_by_name(
+ self.project_name, parent_asset_name),
+ "project_doc": project_doc
+ }
+ )
+
+ self._validate_name_uniqueness(shot_name)
+
+ timing_data = self._get_timing_data(
+ otio_clip,
+ timeline_offset,
+ track_start_frame,
+ workfile_start_frame
+ )
+
+ # create creator attributes
+ creator_attributes = {
+ "asset_name": shot_name,
+ "Parent hierarchy path": shot_metadata["hierarchy"],
+ "workfile_start_frame": workfile_start_frame,
+ "fps": fps,
+ "handle_start": int(handle_start),
+ "handle_end": int(handle_end)
+ }
+ creator_attributes.update(timing_data)
+
+ # create shared new instance data
+ base_instance_data = {
+ "shotName": shot_name,
+ "variant": variant_name,
+
+ # HACK: just for temporal bug workaround
+ # TODO: should loockup shot name for update
+ "asset": parent_asset_name,
+ "task": "",
+
+ "newAssetPublishing": True,
+
+ # parent time properties
+ "trackStartFrame": track_start_frame,
+ "timelineOffset": timeline_offset,
+ # creator_attributes
+ "creator_attributes": creator_attributes
+ }
+ # add hierarchy shot metadata
+ base_instance_data.update(shot_metadata)
+
+ return base_instance_data
+
+ def _get_timing_data(
+ self,
+ otio_clip,
+ timeline_offset,
+ track_start_frame,
+ workfile_start_frame
+ ):
+ """Returning available timing data
+
+ Args:
+ otio_clip (otio.Clip): otio clip object
+ timeline_offset (int): offset value
+ track_start_frame (int): starting frame input
+ workfile_start_frame (int): start frame for shot's workfiles
+
+ Returns:
+ dict: timing metadata
+ """
+ # frame ranges data
+ clip_in = otio_clip.range_in_parent().start_time.value
+ clip_in += track_start_frame
+ clip_out = otio_clip.range_in_parent().end_time_inclusive().value
+ clip_out += track_start_frame
+ self.log.info(f"clip_in: {clip_in} | clip_out: {clip_out}")
+
+ # add offset in case there is any
+ self.log.debug(f"__ timeline_offset: {timeline_offset}")
+ if timeline_offset:
+ clip_in += timeline_offset
+ clip_out += timeline_offset
+
+ clip_duration = otio_clip.duration().value
+ self.log.info(f"clip duration: {clip_duration}")
+
+ source_in = otio_clip.trimmed_range().start_time.value
+ source_out = source_in + clip_duration
+
+ # define starting frame for future shot
+ frame_start = (
+ clip_in if workfile_start_frame is None
+ else workfile_start_frame
+ )
+ frame_end = frame_start + (clip_duration - 1)
+
+ return {
+ "frameStart": int(frame_start),
+ "frameEnd": int(frame_end),
+ "clipIn": int(clip_in),
+ "clipOut": int(clip_out),
+ "clipDuration": int(otio_clip.duration().value),
+ "sourceIn": int(source_in),
+ "sourceOut": int(source_out)
+ }
+
+ def _get_allowed_family_presets(self, pre_create_data):
+ """ Filter out allowed family presets.
+
+ Args:
+ pre_create_data (dict): precreate attributes inputs
+
+ Returns:
+ list: lit of dict with preset items
+ """
+ self.log.debug(f"__ pre_create_data: {pre_create_data}")
+ return [
+ {"family": "shot"},
+ *[
+ preset for preset in self._creator_settings["family_presets"]
+ if pre_create_data[preset["family"]]
+ ]
+ ]
+
+ def _validate_clip_for_processing(self, otio_clip):
+ """Validate otio clip attribues
+
+ Args:
+ otio_clip (otio.Clip): otio clip object
+
+ Returns:
+ bool: True if all passing conditions
+ """
+ if otio_clip.name is None:
+ return False
+
+ if isinstance(otio_clip, otio.schema.Gap):
+ return False
+
+ # skip all generators like black empty
+ if isinstance(
+ otio_clip.media_reference,
+ otio.schema.GeneratorReference):
+ return False
+
+ # Transitions are ignored, because Clips have the full frame
+ # range.
+ if isinstance(otio_clip, otio.schema.Transition):
+ return False
+
+ return True
+
+ def _validate_name_uniqueness(self, name):
+ """ Validating name uniqueness.
+
+ In context of other clip names in sequence file.
+
+ Args:
+ name (str): shot name string
+ """
+ if name not in self.asset_name_check:
+ self.asset_name_check.append(name)
+ else:
+ self.log.warning(
+ f"Duplicate shot name: {name}! "
+ "Please check names in the input sequence files."
+ )
+
+ def get_pre_create_attr_defs(self):
+ """ Creating pre-create attributes at creator plugin.
+
+ Returns:
+ list: list of attribute object instances
+ """
+ # Use same attributes as for instance attrobites
+ attr_defs = [
+ FileDef(
+ "sequence_filepath_data",
+ folders=False,
+ extensions=[
+ ".edl",
+ ".xml",
+ ".aaf",
+ ".fcpxml"
+ ],
+ allow_sequences=False,
+ single_item=True,
+ label="Sequence file",
+ ),
+ FileDef(
+ "media_filepaths_data",
+ folders=False,
+ extensions=[
+ ".mov",
+ ".mp4",
+ ".wav"
+ ],
+ allow_sequences=False,
+ single_item=False,
+ label="Media files",
+ ),
+ # TODO: perhpas better would be timecode and fps input
+ NumberDef(
+ "timeline_offset",
+ default=0,
+ label="Timeline offset"
+ ),
+ UISeparatorDef(),
+ UILabelDef("Clip instance attributes"),
+ UISeparatorDef()
+ ]
+ # add variants swithers
+ attr_defs.extend(
+ BoolDef(_var["family"], label=_var["family"])
+ for _var in self._creator_settings["family_presets"]
+ )
+ attr_defs.append(UISeparatorDef())
+
+ attr_defs.extend(CLIP_ATTR_DEFS)
+ return attr_defs
diff --git a/openpype/hosts/traypublisher/plugins/create/create_from_settings.py b/openpype/hosts/traypublisher/plugins/create/create_from_settings.py
new file mode 100644
index 00000000000..41c1c29bb02
--- /dev/null
+++ b/openpype/hosts/traypublisher/plugins/create/create_from_settings.py
@@ -0,0 +1,22 @@
+import os
+from openpype.api import get_project_settings, Logger
+
+log = Logger.get_logger(__name__)
+
+
+def initialize():
+ from openpype.hosts.traypublisher.api.plugin import SettingsCreator
+
+ project_name = os.environ["AVALON_PROJECT"]
+ project_settings = get_project_settings(project_name)
+
+ simple_creators = project_settings["traypublisher"]["simple_creators"]
+
+ global_variables = globals()
+ for item in simple_creators:
+
+ dynamic_plugin = SettingsCreator.from_settings(item)
+ global_variables[dynamic_plugin.__name__] = dynamic_plugin
+
+
+initialize()
diff --git a/openpype/hosts/traypublisher/plugins/create/create_movie_batch.py b/openpype/hosts/traypublisher/plugins/create/create_movie_batch.py
new file mode 100644
index 00000000000..abe29d74736
--- /dev/null
+++ b/openpype/hosts/traypublisher/plugins/create/create_movie_batch.py
@@ -0,0 +1,218 @@
+import copy
+import os
+import re
+
+from openpype.client import get_assets, get_asset_by_name
+from openpype.lib import (
+ FileDef,
+ BoolDef,
+)
+from openpype.pipeline import (
+ CreatedInstance,
+ CreatorError
+)
+from openpype.pipeline.create import (
+ get_subset_name,
+ TaskNotSetError,
+)
+
+from openpype.hosts.traypublisher.api.plugin import TrayPublishCreator
+
+
+class BatchMovieCreator(TrayPublishCreator):
+ """Creates instances from movie file(s).
+
+ Intended for .mov files, but should work for any video file.
+ Doesn't handle image sequences though.
+ """
+ identifier = "render_movie_batch"
+ label = "Batch Movies"
+ family = "render"
+ description = "Publish batch of video files"
+
+ create_allow_context_change = False
+ version_regex = re.compile(r"^(.+)_v([0-9]+)$")
+
+ def __init__(self, project_settings, *args, **kwargs):
+ super(BatchMovieCreator, self).__init__(project_settings,
+ *args, **kwargs)
+ creator_settings = (
+ project_settings["traypublisher"]["BatchMovieCreator"]
+ )
+ self.default_variants = creator_settings["default_variants"]
+ self.default_tasks = creator_settings["default_tasks"]
+ self.extensions = creator_settings["extensions"]
+
+ def get_icon(self):
+ return "fa.file"
+
+ def create(self, subset_name, data, pre_create_data):
+ file_paths = pre_create_data.get("filepath")
+ if not file_paths:
+ return
+
+ for file_info in file_paths:
+ instance_data = copy.deepcopy(data)
+ file_name = file_info["filenames"][0]
+ filepath = os.path.join(file_info["directory"], file_name)
+ instance_data["creator_attributes"] = {"filepath": filepath}
+
+ asset_doc, version = self.get_asset_doc_from_file_name(
+ file_name, self.project_name)
+
+ subset_name, task_name = self._get_subset_and_task(
+ asset_doc, data["variant"], self.project_name)
+
+ instance_data["task"] = task_name
+ instance_data["asset"] = asset_doc["name"]
+
+ # Create new instance
+ new_instance = CreatedInstance(self.family, subset_name,
+ instance_data, self)
+ self._store_new_instance(new_instance)
+
+ def get_asset_doc_from_file_name(self, source_filename, project_name):
+ """Try to parse out asset name from file name provided.
+
+ Artists might provide various file name formats.
+ Currently handled:
+ - chair.mov
+ - chair_v001.mov
+ - my_chair_to_upload.mov
+ """
+ version = None
+ asset_name = os.path.splitext(source_filename)[0]
+ # Always first check if source filename is in assets
+ matching_asset_doc = self._get_asset_by_name_case_not_sensitive(
+ project_name, asset_name)
+
+ if matching_asset_doc is None:
+ matching_asset_doc, version = (
+ self._parse_with_version(project_name, asset_name))
+
+ if matching_asset_doc is None:
+ matching_asset_doc = self._parse_containing(project_name,
+ asset_name)
+
+ if matching_asset_doc is None:
+ raise CreatorError(
+ "Cannot guess asset name from {}".format(source_filename))
+
+ return matching_asset_doc, version
+
+ def _parse_with_version(self, project_name, asset_name):
+ """Try to parse asset name from a file name containing version too
+
+ Eg. 'chair_v001.mov' >> 'chair', 1
+ """
+ self.log.debug((
+ "Asset doc by \"{}\" was not found, trying version regex."
+ ).format(asset_name))
+
+ matching_asset_doc = version_number = None
+
+ regex_result = self.version_regex.findall(asset_name)
+ if regex_result:
+ _asset_name, _version_number = regex_result[0]
+ matching_asset_doc = self._get_asset_by_name_case_not_sensitive(
+ project_name, _asset_name)
+ if matching_asset_doc:
+ version_number = int(_version_number)
+
+ return matching_asset_doc, version_number
+
+ def _parse_containing(self, project_name, asset_name):
+ """Look if file name contains any existing asset name"""
+ for asset_doc in get_assets(project_name, fields=["name"]):
+ if asset_doc["name"].lower() in asset_name.lower():
+ return get_asset_by_name(project_name, asset_doc["name"])
+
+ def _get_subset_and_task(self, asset_doc, variant, project_name):
+ """Create subset name according to standard template process"""
+ task_name = self._get_task_name(asset_doc)
+
+ try:
+ subset_name = get_subset_name(
+ self.family,
+ variant,
+ task_name,
+ asset_doc,
+ project_name
+ )
+ except TaskNotSetError:
+ # Create instance with fake task
+ # - instance will be marked as invalid so it can't be published
+ # but user have ability to change it
+ # NOTE: This expect that there is not task 'Undefined' on asset
+ task_name = "Undefined"
+ subset_name = get_subset_name(
+ self.family,
+ variant,
+ task_name,
+ asset_doc,
+ project_name
+ )
+
+ return subset_name, task_name
+
+ def _get_task_name(self, asset_doc):
+ """Get applicable task from 'asset_doc' """
+ available_task_names = {}
+ asset_tasks = asset_doc.get("data", {}).get("tasks") or {}
+ for task_name in asset_tasks.keys():
+ available_task_names[task_name.lower()] = task_name
+
+ task_name = None
+ for _task_name in self.default_tasks:
+ _task_name_low = _task_name.lower()
+ if _task_name_low in available_task_names:
+ task_name = available_task_names[_task_name_low]
+ break
+
+ return task_name
+
+ def get_instance_attr_defs(self):
+ return [
+ BoolDef(
+ "add_review_family",
+ default=True,
+ label="Review"
+ )
+ ]
+
+ def get_pre_create_attr_defs(self):
+ # Use same attributes as for instance attributes
+ return [
+ FileDef(
+ "filepath",
+ folders=False,
+ single_item=False,
+ extensions=self.extensions,
+ label="Filepath"
+ ),
+ BoolDef(
+ "add_review_family",
+ default=True,
+ label="Review"
+ )
+ ]
+
+ def get_detail_description(self):
+ return """# Publish batch of .mov to multiple assets.
+
+ File names must then contain only asset name, or asset name + version.
+ (eg. 'chair.mov', 'chair_v001.mov', not really safe `my_chair_v001.mov`
+ """
+
+ def _get_asset_by_name_case_not_sensitive(self, project_name, asset_name):
+ """Handle more cases in file names"""
+ asset_name = re.compile(asset_name, re.IGNORECASE)
+
+ assets = list(get_assets(project_name, asset_names=[asset_name]))
+ if assets:
+ if len(assets) > 1:
+ self.log.warning("Too many records found for {}".format(
+ asset_name))
+ return
+
+ return assets.pop()
diff --git a/openpype/hosts/traypublisher/plugins/publish/collect_app_name.py b/openpype/hosts/traypublisher/plugins/publish/collect_app_name.py
new file mode 100644
index 00000000000..e38d10e70fd
--- /dev/null
+++ b/openpype/hosts/traypublisher/plugins/publish/collect_app_name.py
@@ -0,0 +1,13 @@
+import pyblish.api
+
+
+class CollectTrayPublisherAppName(pyblish.api.ContextPlugin):
+ """Collect app name and label."""
+
+ label = "Collect App Name/Label"
+ order = pyblish.api.CollectorOrder - 0.5
+ hosts = ["traypublisher"]
+
+ def process(self, context):
+ context.data["appName"] = "tray publisher"
+ context.data["appLabel"] = "Tray publisher"
diff --git a/openpype/hosts/traypublisher/plugins/publish/collect_clip_instances.py b/openpype/hosts/traypublisher/plugins/publish/collect_clip_instances.py
new file mode 100644
index 00000000000..bdf7c05f3de
--- /dev/null
+++ b/openpype/hosts/traypublisher/plugins/publish/collect_clip_instances.py
@@ -0,0 +1,36 @@
+from pprint import pformat
+import pyblish.api
+
+
+class CollectClipInstance(pyblish.api.InstancePlugin):
+ """Collect clip instances and resolve its parent"""
+
+ label = "Collect Clip Instances"
+ order = pyblish.api.CollectorOrder - 0.081
+
+ hosts = ["traypublisher"]
+ families = ["plate", "review", "audio"]
+
+ def process(self, instance):
+ creator_identifier = instance.data["creator_identifier"]
+ if creator_identifier not in [
+ "editorial_plate",
+ "editorial_audio",
+ "editorial_review"
+ ]:
+ return
+
+ instance.data["families"].append("clip")
+
+ parent_instance_id = instance.data["parent_instance_id"]
+ edit_shared_data = instance.context.data["editorialSharedData"]
+ instance.data.update(
+ edit_shared_data[parent_instance_id]
+ )
+
+ if "editorialSourcePath" in instance.context.data.keys():
+ instance.data["editorialSourcePath"] = (
+ instance.context.data["editorialSourcePath"])
+ instance.data["families"].append("trimming")
+
+ self.log.debug(pformat(instance.data))
diff --git a/openpype/hosts/traypublisher/plugins/publish/collect_editorial_instances.py b/openpype/hosts/traypublisher/plugins/publish/collect_editorial_instances.py
new file mode 100644
index 00000000000..e181d0abe51
--- /dev/null
+++ b/openpype/hosts/traypublisher/plugins/publish/collect_editorial_instances.py
@@ -0,0 +1,48 @@
+import os
+from pprint import pformat
+import pyblish.api
+import opentimelineio as otio
+
+
+class CollectEditorialInstance(pyblish.api.InstancePlugin):
+ """Collect data for instances created by settings creators."""
+
+ label = "Collect Editorial Instances"
+ order = pyblish.api.CollectorOrder - 0.1
+
+ hosts = ["traypublisher"]
+ families = ["editorial"]
+
+ def process(self, instance):
+
+ if "families" not in instance.data:
+ instance.data["families"] = []
+
+ if "representations" not in instance.data:
+ instance.data["representations"] = []
+
+ fpath = instance.data["sequenceFilePath"]
+ otio_timeline_string = instance.data.pop("otioTimeline")
+ otio_timeline = otio.adapters.read_from_string(
+ otio_timeline_string)
+
+ instance.context.data["otioTimeline"] = otio_timeline
+ instance.context.data["editorialSourcePath"] = (
+ instance.data["editorialSourcePath"])
+
+ self.log.info(fpath)
+
+ instance.data["stagingDir"] = os.path.dirname(fpath)
+
+ _, ext = os.path.splitext(fpath)
+
+ instance.data["representations"].append({
+ "ext": ext[1:],
+ "name": ext[1:],
+ "stagingDir": instance.data["stagingDir"],
+ "files": os.path.basename(fpath)
+ })
+
+ self.log.debug("Created Editorial Instance {}".format(
+ pformat(instance.data)
+ ))
diff --git a/openpype/hosts/traypublisher/plugins/publish/collect_editorial_reviewable.py b/openpype/hosts/traypublisher/plugins/publish/collect_editorial_reviewable.py
new file mode 100644
index 00000000000..4af4fb94e94
--- /dev/null
+++ b/openpype/hosts/traypublisher/plugins/publish/collect_editorial_reviewable.py
@@ -0,0 +1,30 @@
+import pyblish.api
+
+
+class CollectEditorialReviewable(pyblish.api.InstancePlugin):
+ """ Collect review input from user.
+
+ Adds the input to instance data.
+ """
+
+ label = "Collect Editorial Reviewable"
+ order = pyblish.api.CollectorOrder
+
+ families = ["plate", "review", "audio"]
+ hosts = ["traypublisher"]
+
+ def process(self, instance):
+ creator_identifier = instance.data["creator_identifier"]
+ if creator_identifier not in [
+ "editorial_plate",
+ "editorial_audio",
+ "editorial_review"
+ ]:
+ return
+
+ creator_attributes = instance.data["creator_attributes"]
+
+ if creator_attributes["add_review_family"]:
+ instance.data["families"].append("review")
+
+ self.log.debug("instance.data {}".format(instance.data))
diff --git a/openpype/hosts/traypublisher/plugins/publish/collect_movie_batch.py b/openpype/hosts/traypublisher/plugins/publish/collect_movie_batch.py
new file mode 100644
index 00000000000..f37e04d1c91
--- /dev/null
+++ b/openpype/hosts/traypublisher/plugins/publish/collect_movie_batch.py
@@ -0,0 +1,47 @@
+import os
+
+import pyblish.api
+from openpype.pipeline import OpenPypePyblishPluginMixin
+
+
+class CollectMovieBatch(
+ pyblish.api.InstancePlugin, OpenPypePyblishPluginMixin
+):
+ """Collect file url for batch movies and create representation.
+
+ Adds review on instance and to repre.tags based on value of toggle button
+ on creator.
+ """
+
+ label = "Collect Movie Batch Files"
+ order = pyblish.api.CollectorOrder
+
+ hosts = ["traypublisher"]
+
+ def process(self, instance):
+ if instance.data.get("creator_identifier") != "render_movie_batch":
+ return
+
+ creator_attributes = instance.data["creator_attributes"]
+
+ file_url = creator_attributes["filepath"]
+ file_name = os.path.basename(file_url)
+ _, ext = os.path.splitext(file_name)
+
+ repre = {
+ "name": ext[1:],
+ "ext": ext[1:],
+ "files": file_name,
+ "stagingDir": os.path.dirname(file_url),
+ "tags": []
+ }
+
+ if creator_attributes["add_review_family"]:
+ repre["tags"].append("review")
+ instance.data["families"].append("review")
+
+ instance.data["representations"].append(repre)
+
+ instance.data["source"] = file_url
+
+ self.log.debug("instance.data {}".format(instance.data))
diff --git a/openpype/hosts/traypublisher/plugins/publish/collect_shot_instances.py b/openpype/hosts/traypublisher/plugins/publish/collect_shot_instances.py
new file mode 100644
index 00000000000..716f73022e4
--- /dev/null
+++ b/openpype/hosts/traypublisher/plugins/publish/collect_shot_instances.py
@@ -0,0 +1,213 @@
+from pprint import pformat
+import pyblish.api
+import opentimelineio as otio
+
+
+class CollectShotInstance(pyblish.api.InstancePlugin):
+ """ Collect shot instances
+
+ Resolving its user inputs from creator attributes
+ to instance data.
+ """
+
+ label = "Collect Shot Instances"
+ order = pyblish.api.CollectorOrder - 0.09
+
+ hosts = ["traypublisher"]
+ families = ["shot"]
+
+ SHARED_KEYS = [
+ "asset",
+ "fps",
+ "handleStart",
+ "handleEnd",
+ "frameStart",
+ "frameEnd",
+ "clipIn",
+ "clipOut",
+ "clipDuration",
+ "sourceIn",
+ "sourceOut",
+ "otioClip",
+ "workfileFrameStart"
+ ]
+
+ def process(self, instance):
+ self.log.debug(pformat(instance.data))
+
+ creator_identifier = instance.data["creator_identifier"]
+ if "editorial" not in creator_identifier:
+ return
+
+ # get otio clip object
+ otio_clip = self._get_otio_clip(instance)
+ instance.data["otioClip"] = otio_clip
+
+ # first solve the inputs from creator attr
+ data = self._solve_inputs_to_data(instance)
+ instance.data.update(data)
+
+ # distribute all shared keys to clips instances
+ self._distribute_shared_data(instance)
+ self._solve_hierarchy_context(instance)
+
+ self.log.debug(pformat(instance.data))
+
+ def _get_otio_clip(self, instance):
+ """ Converts otio string data.
+
+ Convert them to proper otio object
+ and finds its equivalent at otio timeline.
+ This process is a hack to support also
+ resolving parent range.
+
+ Args:
+ instance (obj): publishing instance
+
+ Returns:
+ otio.Clip: otio clip object
+ """
+ context = instance.context
+ # convert otio clip from string to object
+ otio_clip_string = instance.data.pop("otioClip")
+ otio_clip = otio.adapters.read_from_string(
+ otio_clip_string)
+
+ otio_timeline = context.data["otioTimeline"]
+
+ clips = [
+ clip for clip in otio_timeline.each_child(
+ descended_from_type=otio.schema.Clip)
+ if clip.name == otio_clip.name
+ ]
+
+ otio_clip = clips.pop()
+ self.log.debug(f"__ otioclip.parent: {otio_clip.parent}")
+
+ return otio_clip
+
+ def _distribute_shared_data(self, instance):
+ """ Distribute all defined keys.
+
+ All data are shared between all related
+ instances in context.
+
+ Args:
+ instance (obj): publishing instance
+ """
+ context = instance.context
+
+ instance_id = instance.data["instance_id"]
+
+ if not context.data.get("editorialSharedData"):
+ context.data["editorialSharedData"] = {}
+
+ context.data["editorialSharedData"][instance_id] = {
+ _k: _v for _k, _v in instance.data.items()
+ if _k in self.SHARED_KEYS
+ }
+
+ def _solve_inputs_to_data(self, instance):
+ """ Resolve all user inputs into instance data.
+
+ Args:
+ instance (obj): publishing instance
+
+ Returns:
+ dict: instance data updating data
+ """
+ _cr_attrs = instance.data["creator_attributes"]
+ workfile_start_frame = _cr_attrs["workfile_start_frame"]
+ frame_start = _cr_attrs["frameStart"]
+ frame_end = _cr_attrs["frameEnd"]
+ frame_dur = frame_end - frame_start
+
+ return {
+ "asset": _cr_attrs["asset_name"],
+ "fps": float(_cr_attrs["fps"]),
+ "handleStart": _cr_attrs["handle_start"],
+ "handleEnd": _cr_attrs["handle_end"],
+ "frameStart": workfile_start_frame,
+ "frameEnd": workfile_start_frame + frame_dur,
+ "clipIn": _cr_attrs["clipIn"],
+ "clipOut": _cr_attrs["clipOut"],
+ "clipDuration": _cr_attrs["clipDuration"],
+ "sourceIn": _cr_attrs["sourceIn"],
+ "sourceOut": _cr_attrs["sourceOut"],
+ "workfileFrameStart": workfile_start_frame
+ }
+
+ def _solve_hierarchy_context(self, instance):
+ """ Adding hierarchy data to context shared data.
+
+ Args:
+ instance (obj): publishing instance
+ """
+ context = instance.context
+
+ final_context = (
+ context.data["hierarchyContext"]
+ if context.data.get("hierarchyContext")
+ else {}
+ )
+
+ name = instance.data["asset"]
+
+ # get handles
+ handle_start = int(instance.data["handleStart"])
+ handle_end = int(instance.data["handleEnd"])
+
+ in_info = {
+ "entity_type": "Shot",
+ "custom_attributes": {
+ "handleStart": handle_start,
+ "handleEnd": handle_end,
+ "frameStart": instance.data["frameStart"],
+ "frameEnd": instance.data["frameEnd"],
+ "clipIn": instance.data["clipIn"],
+ "clipOut": instance.data["clipOut"],
+ "fps": instance.data["fps"]
+ },
+ "tasks": instance.data["tasks"]
+ }
+
+ parents = instance.data.get('parents', [])
+ self.log.debug(f"parents: {pformat(parents)}")
+
+ actual = {name: in_info}
+
+ for parent in reversed(parents):
+ parent_name = parent["entity_name"]
+ next_dict = {
+ parent_name: {
+ "entity_type": parent["entity_type"],
+ "childs": actual
+ }
+ }
+ actual = next_dict
+
+ final_context = self._update_dict(final_context, actual)
+
+ # adding hierarchy context to instance
+ context.data["hierarchyContext"] = final_context
+ self.log.debug(pformat(final_context))
+
+ def _update_dict(self, ex_dict, new_dict):
+ """ Recursion function
+
+ Updating nested data with another nested data.
+
+ Args:
+ ex_dict (dict): nested data
+ new_dict (dict): nested data
+
+ Returns:
+ dict: updated nested data
+ """
+ for key in ex_dict:
+ if key in new_dict and isinstance(ex_dict[key], dict):
+ new_dict[key] = self._update_dict(ex_dict[key], new_dict[key])
+ elif not ex_dict.get(key) or not new_dict.get(key):
+ new_dict[key] = ex_dict[key]
+
+ return new_dict
diff --git a/openpype/hosts/traypublisher/plugins/publish/collect_simple_instances.py b/openpype/hosts/traypublisher/plugins/publish/collect_simple_instances.py
new file mode 100644
index 00000000000..c0ae694c3cc
--- /dev/null
+++ b/openpype/hosts/traypublisher/plugins/publish/collect_simple_instances.py
@@ -0,0 +1,228 @@
+import os
+import tempfile
+
+import clique
+import pyblish.api
+
+
+class CollectSettingsSimpleInstances(pyblish.api.InstancePlugin):
+ """Collect data for instances created by settings creators.
+
+ Plugin create representations for simple instances based
+ on 'representation_files' attribute stored on instance data.
+
+ There is also possibility to have reviewable representation which can be
+ stored under 'reviewable' attribute stored on instance data. If there was
+ already created representation with the same files as 'revieable' containes
+
+ Representations can be marked for review and in that case is also added
+ 'review' family to instance families. For review can be marked only one
+ representation so **first** representation that has extension available
+ in '_review_extensions' is used for review.
+
+ For instance 'source' is used path from last representation created
+ from 'representation_files'.
+
+ Set staging directory on instance. That is probably never used because
+ each created representation has it's own staging dir.
+ """
+
+ label = "Collect Settings Simple Instances"
+ order = pyblish.api.CollectorOrder - 0.49
+
+ hosts = ["traypublisher"]
+
+ def process(self, instance):
+ if not instance.data.get("settings_creator"):
+ return
+
+ instance_label = instance.data["name"]
+ # Create instance's staging dir in temp
+ tmp_folder = tempfile.mkdtemp(prefix="traypublisher_")
+ instance.data["stagingDir"] = tmp_folder
+ instance.context.data["cleanupFullPaths"].append(tmp_folder)
+
+ self.log.debug((
+ "Created temp staging directory for instance {}. {}"
+ ).format(instance_label, tmp_folder))
+
+ # Store filepaths for validation of their existence
+ source_filepaths = []
+ # Make sure there are no representations with same name
+ repre_names_counter = {}
+ # Store created names for logging
+ repre_names = []
+ # Store set of filepaths per each representation
+ representation_files_mapping = []
+ source = self._create_main_representations(
+ instance,
+ source_filepaths,
+ repre_names_counter,
+ repre_names,
+ representation_files_mapping
+ )
+
+ self._create_review_representation(
+ instance,
+ source_filepaths,
+ repre_names_counter,
+ repre_names,
+ representation_files_mapping
+ )
+
+ instance.data["source"] = source
+ instance.data["sourceFilepaths"] = list(set(source_filepaths))
+
+ self.log.debug(
+ (
+ "Created Simple Settings instance \"{}\""
+ " with {} representations: {}"
+ ).format(
+ instance_label,
+ len(instance.data["representations"]),
+ ", ".join(repre_names)
+ )
+ )
+
+ def _create_main_representations(
+ self,
+ instance,
+ source_filepaths,
+ repre_names_counter,
+ repre_names,
+ representation_files_mapping
+ ):
+ creator_attributes = instance.data["creator_attributes"]
+ filepath_items = creator_attributes["representation_files"]
+ if not isinstance(filepath_items, list):
+ filepath_items = [filepath_items]
+
+ source = None
+ for filepath_item in filepath_items:
+ # Skip if filepath item does not have filenames
+ if not filepath_item["filenames"]:
+ continue
+
+ filepaths = {
+ os.path.join(filepath_item["directory"], filename)
+ for filename in filepath_item["filenames"]
+ }
+ source_filepaths.extend(filepaths)
+
+ source = self._calculate_source(filepaths)
+ representation = self._create_representation_data(
+ filepath_item, repre_names_counter, repre_names
+ )
+ instance.data["representations"].append(representation)
+ representation_files_mapping.append(
+ (filepaths, representation, source)
+ )
+ return source
+
+ def _create_review_representation(
+ self,
+ instance,
+ source_filepaths,
+ repre_names_counter,
+ repre_names,
+ representation_files_mapping
+ ):
+ # Skip review representation creation if there are no representations
+ # created for "main" part
+ # - review representation must not be created in that case so
+ # validation can care about it
+ if not representation_files_mapping:
+ self.log.warning((
+ "There are missing source representations."
+ " Creation of review representation was skipped."
+ ))
+ return
+
+ creator_attributes = instance.data["creator_attributes"]
+ review_file_item = creator_attributes["reviewable"]
+ filenames = review_file_item.get("filenames")
+ if not filenames:
+ self.log.debug((
+ "Filepath for review is not defined."
+ " Skipping review representation creation."
+ ))
+ return
+
+ filepaths = {
+ os.path.join(review_file_item["directory"], filename)
+ for filename in filenames
+ }
+ source_filepaths.extend(filepaths)
+ # First try to find out representation with same filepaths
+ # so it's not needed to create new representation just for review
+ review_representation = None
+ # Review path (only for logging)
+ review_path = None
+ for item in representation_files_mapping:
+ _filepaths, representation, repre_path = item
+ if _filepaths == filepaths:
+ review_representation = representation
+ review_path = repre_path
+ break
+
+ if review_representation is None:
+ self.log.debug("Creating new review representation")
+ review_path = self._calculate_source(filepaths)
+ review_representation = self._create_representation_data(
+ review_file_item, repre_names_counter, repre_names
+ )
+ instance.data["representations"].append(review_representation)
+
+ if "review" not in instance.data["families"]:
+ instance.data["families"].append("review")
+
+ review_representation["tags"].append("review")
+ self.log.debug("Representation {} was marked for review. {}".format(
+ review_representation["name"], review_path
+ ))
+
+ def _create_representation_data(
+ self, filepath_item, repre_names_counter, repre_names
+ ):
+ """Create new representation data based on file item.
+
+ Args:
+ filepath_item (Dict[str, Any]): Item with information about
+ representation paths.
+ repre_names_counter (Dict[str, int]): Store count of representation
+ names.
+ repre_names (List[str]): All used representation names. For
+ logging purposes.
+
+ Returns:
+ Dict: Prepared base representation data.
+ """
+
+ filenames = filepath_item["filenames"]
+ _, ext = os.path.splitext(filenames[0])
+ if len(filenames) == 1:
+ filenames = filenames[0]
+
+ repre_name = repre_ext = ext[1:]
+ if repre_name not in repre_names_counter:
+ repre_names_counter[repre_name] = 2
+ else:
+ counter = repre_names_counter[repre_name]
+ repre_names_counter[repre_name] += 1
+ repre_name = "{}_{}".format(repre_name, counter)
+ repre_names.append(repre_name)
+ return {
+ "ext": repre_ext,
+ "name": repre_name,
+ "stagingDir": filepath_item["directory"],
+ "files": filenames,
+ "tags": []
+ }
+
+ def _calculate_source(self, filepaths):
+ cols, rems = clique.assemble(filepaths)
+ if cols:
+ source = cols[0].format("{head}{padding}{tail}")
+ elif rems:
+ source = rems[0]
+ return source
diff --git a/openpype/hosts/traypublisher/plugins/publish/collect_source.py b/openpype/hosts/traypublisher/plugins/publish/collect_source.py
new file mode 100644
index 00000000000..6ff22be13a3
--- /dev/null
+++ b/openpype/hosts/traypublisher/plugins/publish/collect_source.py
@@ -0,0 +1,24 @@
+import pyblish.api
+
+
+class CollectSource(pyblish.api.ContextPlugin):
+ """Collecting instances from traypublisher host."""
+
+ label = "Collect source"
+ order = pyblish.api.CollectorOrder - 0.49
+ hosts = ["traypublisher"]
+
+ def process(self, context):
+ # get json paths from os and load them
+ source_name = "traypublisher"
+ for instance in context:
+ source = instance.data.get("source")
+ if not source:
+ instance.data["source"] = source_name
+ self.log.info((
+ "Source of instance \"{}\" is changed to \"{}\""
+ ).format(instance.data["name"], source_name))
+ else:
+ self.log.info((
+ "Source of instance \"{}\" was already set to \"{}\""
+ ).format(instance.data["name"], source))
diff --git a/openpype/hosts/traypublisher/plugins/publish/help/validate_frame_ranges.xml b/openpype/hosts/traypublisher/plugins/publish/help/validate_frame_ranges.xml
new file mode 100644
index 00000000000..933df1c7c5c
--- /dev/null
+++ b/openpype/hosts/traypublisher/plugins/publish/help/validate_frame_ranges.xml
@@ -0,0 +1,15 @@
+
+
+
+Invalid frame range
+
+## Invalid frame range
+
+Expected duration or '{duration}' frames set in database, workfile contains only '{found}' frames.
+
+### How to repair?
+
+Modify configuration in the database or tweak frame range in the workfile.
+
+
+
\ No newline at end of file
diff --git a/openpype/hosts/traypublisher/plugins/publish/validate_filepaths.py b/openpype/hosts/traypublisher/plugins/publish/validate_filepaths.py
new file mode 100644
index 00000000000..749199fbd3a
--- /dev/null
+++ b/openpype/hosts/traypublisher/plugins/publish/validate_filepaths.py
@@ -0,0 +1,68 @@
+import os
+import pyblish.api
+from openpype.pipeline import PublishValidationError
+
+
+class ValidateFilePath(pyblish.api.InstancePlugin):
+ """Validate existence of source filepaths on instance.
+
+ Plugins looks into key 'sourceFilepaths' and validate if paths there
+ actually exist on disk.
+
+ Also validate if the key is filled but is empty. In that case also
+ crashes so do not fill the key if unfilled value should not cause error.
+
+ This is primarily created for Simple Creator instances.
+ """
+
+ label = "Validate Workfile"
+ order = pyblish.api.ValidatorOrder - 0.49
+
+ hosts = ["traypublisher"]
+
+ def process(self, instance):
+ if "sourceFilepaths" not in instance.data:
+ self.log.info((
+ "Skipped validation of source filepaths existence."
+ " Instance does not have collected 'sourceFilepaths'"
+ ))
+ return
+
+ family = instance.data["family"]
+ label = instance.data["name"]
+ filepaths = instance.data["sourceFilepaths"]
+ if not filepaths:
+ raise PublishValidationError(
+ (
+ "Source filepaths of '{}' instance \"{}\" are not filled"
+ ).format(family, label),
+ "File not filled",
+ (
+ "## Files were not filled"
+ "\nThis mean that you didn't enter any files into required"
+ " file input."
+ "\n- Please refresh publishing and check instance"
+ " {} "
+ ).format(label)
+ )
+
+ not_found_files = [
+ filepath
+ for filepath in filepaths
+ if not os.path.exists(filepath)
+ ]
+ if not_found_files:
+ joined_paths = "\n".join([
+ "- {}".format(filepath)
+ for filepath in not_found_files
+ ])
+ raise PublishValidationError(
+ (
+ "Filepath of '{}' instance \"{}\" does not exist:\n{}"
+ ).format(family, label, joined_paths),
+ "File not found",
+ (
+ "## Files were not found\nFiles\n{}"
+ "\n\nCheck if the path is still available."
+ ).format(joined_paths)
+ )
diff --git a/openpype/hosts/traypublisher/plugins/publish/validate_frame_ranges.py b/openpype/hosts/traypublisher/plugins/publish/validate_frame_ranges.py
new file mode 100644
index 00000000000..b962ea464a2
--- /dev/null
+++ b/openpype/hosts/traypublisher/plugins/publish/validate_frame_ranges.py
@@ -0,0 +1,75 @@
+import re
+
+import pyblish.api
+
+from openpype.pipeline.publish import (
+ ValidateContentsOrder,
+ PublishXmlValidationError,
+ OptionalPyblishPluginMixin,
+)
+
+
+class ValidateFrameRange(OptionalPyblishPluginMixin,
+ pyblish.api.InstancePlugin):
+ """Validating frame range of rendered files against state in DB."""
+
+ label = "Validate Frame Range"
+ hosts = ["traypublisher"]
+ families = ["render"]
+ order = ValidateContentsOrder
+
+ optional = True
+ # published data might be sequence (.mov, .mp4) in that counting files
+ # doesnt make sense
+ check_extensions = ["exr", "dpx", "jpg", "jpeg", "png", "tiff", "tga",
+ "gif", "svg"]
+ skip_timelines_check = [] # skip for specific task names (regex)
+
+ def process(self, instance):
+ # Skip the instance if is not active by data on the instance
+ if not self.is_active(instance.data):
+ return
+
+ if (self.skip_timelines_check and
+ any(re.search(pattern, instance.data["task"])
+ for pattern in self.skip_timelines_check)):
+ self.log.info("Skipping for {} task".format(instance.data["task"]))
+
+ asset_doc = instance.data["assetEntity"]
+ asset_data = asset_doc["data"]
+ frame_start = asset_data["frameStart"]
+ frame_end = asset_data["frameEnd"]
+ handle_start = asset_data["handleStart"]
+ handle_end = asset_data["handleEnd"]
+ duration = (frame_end - frame_start + 1) + handle_start + handle_end
+
+ repres = instance.data.get("representations")
+ if not repres:
+ self.log.info("No representations, skipping.")
+ return
+
+ first_repre = repres[0]
+ ext = first_repre['ext'].replace(".", '')
+
+ if not ext or ext.lower() not in self.check_extensions:
+ self.log.warning("Cannot check for extension {}".format(ext))
+ return
+
+ files = first_repre["files"]
+ if isinstance(files, str):
+ files = [files]
+ frames = len(files)
+
+ msg = (
+ "Frame duration from DB:'{}' doesn't match number of files:'{}'"
+ " Please change frame range for Asset or limit no. of files"
+ ). format(int(duration), frames)
+
+ formatting_data = {"duration": duration,
+ "found": frames}
+ if frames != duration:
+ raise PublishXmlValidationError(self, msg,
+ formatting_data=formatting_data)
+
+ self.log.debug("Valid ranges expected '{}' - found '{}'".
+ format(int(duration), frames))
diff --git a/openpype/hosts/tvpaint/__init__.py b/openpype/hosts/tvpaint/__init__.py
index 0e793fcf9f3..b98680f2045 100644
--- a/openpype/hosts/tvpaint/__init__.py
+++ b/openpype/hosts/tvpaint/__init__.py
@@ -1,8 +1,12 @@
-def add_implementation_envs(env, _app):
- """Modify environments to contain all required for implementation."""
- defaults = {
- "OPENPYPE_LOG_NO_COLORS": "True"
- }
- for key, value in defaults.items():
- if not env.get(key):
- env[key] = value
+from .addon import (
+ get_launch_script_path,
+ TVPaintAddon,
+ TVPAINT_ROOT_DIR,
+)
+
+
+__all__ = (
+ "get_launch_script_path",
+ "TVPaintAddon",
+ "TVPAINT_ROOT_DIR",
+)
diff --git a/openpype/hosts/tvpaint/addon.py b/openpype/hosts/tvpaint/addon.py
new file mode 100644
index 00000000000..d710e63f933
--- /dev/null
+++ b/openpype/hosts/tvpaint/addon.py
@@ -0,0 +1,41 @@
+import os
+from openpype.modules import OpenPypeModule
+from openpype.modules.interfaces import IHostAddon
+
+TVPAINT_ROOT_DIR = os.path.dirname(os.path.abspath(__file__))
+
+
+def get_launch_script_path():
+ return os.path.join(
+ TVPAINT_ROOT_DIR,
+ "api",
+ "launch_script.py"
+ )
+
+
+class TVPaintAddon(OpenPypeModule, IHostAddon):
+ name = "tvpaint"
+ host_name = "tvpaint"
+
+ def initialize(self, module_settings):
+ self.enabled = True
+
+ def add_implementation_envs(self, env, _app):
+ """Modify environments to contain all required for implementation."""
+
+ defaults = {
+ "OPENPYPE_LOG_NO_COLORS": "True"
+ }
+ for key, value in defaults.items():
+ if not env.get(key):
+ env[key] = value
+
+ def get_launch_hook_paths(self, app):
+ if app.host_name != self.host_name:
+ return []
+ return [
+ os.path.join(TVPAINT_ROOT_DIR, "hooks")
+ ]
+
+ def get_workfile_extensions(self):
+ return [".tvpp"]
diff --git a/openpype/hosts/tvpaint/api/__init__.py b/openpype/hosts/tvpaint/api/__init__.py
index 1c50987d6d5..43d411d8f90 100644
--- a/openpype/hosts/tvpaint/api/__init__.py
+++ b/openpype/hosts/tvpaint/api/__init__.py
@@ -1,93 +1,47 @@
-import os
-import logging
-
-import requests
-
-import avalon.api
-import pyblish.api
-from avalon.tvpaint import pipeline
-from avalon.tvpaint.communication_server import register_localization_file
-from .lib import set_context_settings
-
-from openpype.hosts import tvpaint
-from openpype.api import get_current_project_settings
-
-log = logging.getLogger(__name__)
-
-HOST_DIR = os.path.dirname(os.path.abspath(tvpaint.__file__))
-PLUGINS_DIR = os.path.join(HOST_DIR, "plugins")
-PUBLISH_PATH = os.path.join(PLUGINS_DIR, "publish")
-LOAD_PATH = os.path.join(PLUGINS_DIR, "load")
-CREATE_PATH = os.path.join(PLUGINS_DIR, "create")
-
-
-def on_instance_toggle(instance, old_value, new_value):
- # Review may not have real instance in wokrfile metadata
- if not instance.data.get("uuid"):
- return
-
- instance_id = instance.data["uuid"]
- found_idx = None
- current_instances = pipeline.list_instances()
- for idx, workfile_instance in enumerate(current_instances):
- if workfile_instance["uuid"] == instance_id:
- found_idx = idx
- break
-
- if found_idx is None:
- return
-
- if "active" in current_instances[found_idx]:
- current_instances[found_idx]["active"] = new_value
- pipeline._write_instances(current_instances)
-
-
-def initial_launch():
- # Setup project settings if its the template that's launched.
- # TODO also check for template creation when it's possible to define
- # templates
- last_workfile = os.environ.get("AVALON_LAST_WORKFILE")
- if not last_workfile or os.path.exists(last_workfile):
- return
-
- log.info("Setting up project...")
- set_context_settings()
-
-
-def application_exit():
- data = get_current_project_settings()
- stop_timer = data["tvpaint"]["stop_timer_on_application_exit"]
-
- if not stop_timer:
- return
-
- # Stop application timer.
- webserver_url = os.environ.get("OPENPYPE_WEBSERVER_URL")
- rest_api_url = "{}/timers_manager/stop_timer".format(webserver_url)
- requests.post(rest_api_url)
-
-
-def install():
- log.info("OpenPype - Installing TVPaint integration")
- localization_file = os.path.join(HOST_DIR, "resources", "avalon.loc")
- register_localization_file(localization_file)
-
- pyblish.api.register_plugin_path(PUBLISH_PATH)
- avalon.api.register_plugin_path(avalon.api.Loader, LOAD_PATH)
- avalon.api.register_plugin_path(avalon.api.Creator, CREATE_PATH)
-
- registered_callbacks = (
- pyblish.api.registered_callbacks().get("instanceToggled") or []
- )
- if on_instance_toggle not in registered_callbacks:
- pyblish.api.register_callback("instanceToggled", on_instance_toggle)
-
- avalon.api.on("application.launched", initial_launch)
- avalon.api.on("application.exit", application_exit)
-
-
-def uninstall():
- log.info("OpenPype - Uninstalling TVPaint integration")
- pyblish.api.deregister_plugin_path(PUBLISH_PATH)
- avalon.api.deregister_plugin_path(avalon.api.Loader, LOAD_PATH)
- avalon.api.deregister_plugin_path(avalon.api.Creator, CREATE_PATH)
+from .communication_server import CommunicationWrapper
+from . import lib
+from . import launch_script
+from . import workio
+from . import pipeline
+from . import plugin
+from .pipeline import (
+ install,
+ maintained_selection,
+ remove_instance,
+ list_instances,
+ ls
+)
+
+from .workio import (
+ open_file,
+ save_file,
+ current_file,
+ has_unsaved_changes,
+ file_extensions,
+ work_root,
+)
+
+
+__all__ = (
+ "CommunicationWrapper",
+
+ "lib",
+ "launch_script",
+ "workio",
+ "pipeline",
+ "plugin",
+
+ "install",
+ "maintained_selection",
+ "remove_instance",
+ "list_instances",
+ "ls",
+
+ # Workfiles API
+ "open_file",
+ "save_file",
+ "current_file",
+ "has_unsaved_changes",
+ "file_extensions",
+ "work_root"
+)
diff --git a/openpype/hosts/tvpaint/api/communication_server.py b/openpype/hosts/tvpaint/api/communication_server.py
new file mode 100644
index 00000000000..6ac3e6324c2
--- /dev/null
+++ b/openpype/hosts/tvpaint/api/communication_server.py
@@ -0,0 +1,945 @@
+import os
+import json
+import time
+import subprocess
+import collections
+import asyncio
+import logging
+import socket
+import platform
+import filecmp
+import tempfile
+import threading
+import shutil
+from queue import Queue
+from contextlib import closing
+
+from aiohttp import web
+from aiohttp_json_rpc import JsonRpc
+from aiohttp_json_rpc.protocol import (
+ encode_request, encode_error, decode_msg, JsonRpcMsgTyp
+)
+from aiohttp_json_rpc.exceptions import RpcError
+
+from openpype.lib import emit_event
+from openpype.hosts.tvpaint.tvpaint_plugin import get_plugin_files_path
+
+log = logging.getLogger(__name__)
+log.setLevel(logging.DEBUG)
+
+
+class CommunicationWrapper:
+ # TODO add logs and exceptions
+ communicator = None
+
+ log = logging.getLogger("CommunicationWrapper")
+
+ @classmethod
+ def create_qt_communicator(cls, *args, **kwargs):
+ """Create communicator for Artist usage."""
+ communicator = QtCommunicator(*args, **kwargs)
+ cls.set_communicator(communicator)
+ return communicator
+
+ @classmethod
+ def set_communicator(cls, communicator):
+ if not cls.communicator:
+ cls.communicator = communicator
+ else:
+ cls.log.warning("Communicator was set multiple times.")
+
+ @classmethod
+ def client(cls):
+ if not cls.communicator:
+ return None
+ return cls.communicator.client()
+
+ @classmethod
+ def execute_george(cls, george_script):
+ """Execute passed goerge script in TVPaint."""
+ if not cls.communicator:
+ return
+ return cls.communicator.execute_george(george_script)
+
+
+class WebSocketServer:
+ def __init__(self):
+ self.client = None
+
+ self.loop = asyncio.new_event_loop()
+ self.app = web.Application(loop=self.loop)
+ self.port = self.find_free_port()
+ self.websocket_thread = WebsocketServerThread(
+ self, self.port, loop=self.loop
+ )
+
+ @property
+ def server_is_running(self):
+ return self.websocket_thread.server_is_running
+
+ def add_route(self, *args, **kwargs):
+ self.app.router.add_route(*args, **kwargs)
+
+ @staticmethod
+ def find_free_port():
+ with closing(
+ socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+ ) as sock:
+ sock.bind(("", 0))
+ sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+ port = sock.getsockname()[1]
+ return port
+
+ def start(self):
+ self.websocket_thread.start()
+
+ def stop(self):
+ try:
+ if self.websocket_thread.is_running:
+ log.debug("Stopping websocket server")
+ self.websocket_thread.is_running = False
+ self.websocket_thread.stop()
+ except Exception:
+ log.warning(
+ "Error has happened during Killing websocket server",
+ exc_info=True
+ )
+
+
+class WebsocketServerThread(threading.Thread):
+ """ Listener for websocket rpc requests.
+
+ It would be probably better to "attach" this to main thread (as for
+ example Harmony needs to run something on main thread), but currently
+ it creates separate thread and separate asyncio event loop
+ """
+ def __init__(self, module, port, loop):
+ super(WebsocketServerThread, self).__init__()
+ self.is_running = False
+ self.server_is_running = False
+ self.port = port
+ self.module = module
+ self.loop = loop
+ self.runner = None
+ self.site = None
+ self.tasks = []
+
+ def run(self):
+ self.is_running = True
+
+ try:
+ log.debug("Starting websocket server")
+
+ self.loop.run_until_complete(self.start_server())
+
+ log.info(
+ "Running Websocket server on URL:"
+ " \"ws://localhost:{}\"".format(self.port)
+ )
+
+ asyncio.ensure_future(self.check_shutdown(), loop=self.loop)
+
+ self.server_is_running = True
+ self.loop.run_forever()
+
+ except Exception:
+ log.warning(
+ "Websocket Server service has failed", exc_info=True
+ )
+ finally:
+ self.server_is_running = False
+ # optional
+ self.loop.close()
+
+ self.is_running = False
+ log.info("Websocket server stopped")
+
+ async def start_server(self):
+ """ Starts runner and TCPsite """
+ self.runner = web.AppRunner(self.module.app)
+ await self.runner.setup()
+ self.site = web.TCPSite(self.runner, "localhost", self.port)
+ await self.site.start()
+
+ def stop(self):
+ """Sets is_running flag to false, 'check_shutdown' shuts server down"""
+ self.is_running = False
+
+ async def check_shutdown(self):
+ """ Future that is running and checks if server should be running
+ periodically.
+ """
+ while self.is_running:
+ while self.tasks:
+ task = self.tasks.pop(0)
+ log.debug("waiting for task {}".format(task))
+ await task
+ log.debug("returned value {}".format(task.result))
+
+ await asyncio.sleep(0.5)
+
+ log.debug("## Server shutdown started")
+
+ await self.site.stop()
+ log.debug("# Site stopped")
+ await self.runner.cleanup()
+ log.debug("# Server runner stopped")
+ tasks = [
+ task for task in asyncio.all_tasks()
+ if task is not asyncio.current_task()
+ ]
+ list(map(lambda task: task.cancel(), tasks)) # cancel all the tasks
+ results = await asyncio.gather(*tasks, return_exceptions=True)
+ log.debug(f"Finished awaiting cancelled tasks, results: {results}...")
+ await self.loop.shutdown_asyncgens()
+ # to really make sure everything else has time to stop
+ await asyncio.sleep(0.07)
+ self.loop.stop()
+
+
+class BaseTVPaintRpc(JsonRpc):
+ def __init__(self, communication_obj, route_name="", **kwargs):
+ super().__init__(**kwargs)
+ self.requests_ids = collections.defaultdict(lambda: 0)
+ self.waiting_requests = collections.defaultdict(list)
+ self.responses = collections.defaultdict(list)
+
+ self.route_name = route_name
+ self.communication_obj = communication_obj
+
+ async def _handle_rpc_msg(self, http_request, raw_msg):
+ # This is duplicated code from super but there is no way how to do it
+ # to be able handle server->client requests
+ host = http_request.host
+ if host in self.waiting_requests:
+ try:
+ _raw_message = raw_msg.data
+ msg = decode_msg(_raw_message)
+
+ except RpcError as error:
+ await self._ws_send_str(http_request, encode_error(error))
+ return
+
+ if msg.type in (JsonRpcMsgTyp.RESULT, JsonRpcMsgTyp.ERROR):
+ msg_data = json.loads(_raw_message)
+ if msg_data.get("id") in self.waiting_requests[host]:
+ self.responses[host].append(msg_data)
+ return
+
+ return await super()._handle_rpc_msg(http_request, raw_msg)
+
+ def client_connected(self):
+ # TODO This is poor check. Add check it is client from TVPaint
+ if self.clients:
+ return True
+ return False
+
+ def send_notification(self, client, method, params=None):
+ if params is None:
+ params = []
+ asyncio.run_coroutine_threadsafe(
+ client.ws.send_str(encode_request(method, params=params)),
+ loop=self.loop
+ )
+
+ def send_request(self, client, method, params=None, timeout=0):
+ if params is None:
+ params = []
+
+ client_host = client.host
+
+ request_id = self.requests_ids[client_host]
+ self.requests_ids[client_host] += 1
+
+ self.waiting_requests[client_host].append(request_id)
+
+ log.debug("Sending request to client {} ({}, {}) id: {}".format(
+ client_host, method, params, request_id
+ ))
+ future = asyncio.run_coroutine_threadsafe(
+ client.ws.send_str(encode_request(method, request_id, params)),
+ loop=self.loop
+ )
+ result = future.result()
+
+ not_found = object()
+ response = not_found
+ start = time.time()
+ while True:
+ if client.ws.closed:
+ return None
+
+ for _response in self.responses[client_host]:
+ _id = _response.get("id")
+ if _id == request_id:
+ response = _response
+ break
+
+ if response is not not_found:
+ break
+
+ if timeout > 0 and (time.time() - start) > timeout:
+ raise Exception("Timeout passed")
+ return
+
+ time.sleep(0.1)
+
+ if response is not_found:
+ raise Exception("Connection closed")
+
+ self.responses[client_host].remove(response)
+
+ error = response.get("error")
+ result = response.get("result")
+ if error:
+ raise Exception("Error happened: {}".format(error))
+ return result
+
+
+class QtTVPaintRpc(BaseTVPaintRpc):
+ def __init__(self, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+
+ from openpype.tools.utils import host_tools
+ self.tools_helper = host_tools.HostToolsHelper()
+
+ route_name = self.route_name
+
+ # Register methods
+ self.add_methods(
+ (route_name, self.workfiles_tool),
+ (route_name, self.loader_tool),
+ (route_name, self.creator_tool),
+ (route_name, self.subset_manager_tool),
+ (route_name, self.publish_tool),
+ (route_name, self.scene_inventory_tool),
+ (route_name, self.library_loader_tool),
+ (route_name, self.experimental_tools)
+ )
+
+ # Panel routes for tools
+ async def workfiles_tool(self):
+ log.info("Triggering Workfile tool")
+ item = MainThreadItem(self.tools_helper.show_workfiles)
+ self._execute_in_main_thread(item)
+ return
+
+ async def loader_tool(self):
+ log.info("Triggering Loader tool")
+ item = MainThreadItem(self.tools_helper.show_loader)
+ self._execute_in_main_thread(item)
+ return
+
+ async def creator_tool(self):
+ log.info("Triggering Creator tool")
+ item = MainThreadItem(self.tools_helper.show_creator)
+ await self._async_execute_in_main_thread(item, wait=False)
+
+ async def subset_manager_tool(self):
+ log.info("Triggering Subset Manager tool")
+ item = MainThreadItem(self.tools_helper.show_subset_manager)
+ # Do not wait for result of callback
+ self._execute_in_main_thread(item, wait=False)
+ return
+
+ async def publish_tool(self):
+ log.info("Triggering Publish tool")
+ item = MainThreadItem(self.tools_helper.show_publish)
+ self._execute_in_main_thread(item)
+ return
+
+ async def scene_inventory_tool(self):
+ """Open Scene Inventory tool.
+
+ Function can't confirm if tool was opened becauise one part of
+ SceneInventory initialization is calling websocket request to host but
+ host can't response because is waiting for response from this call.
+ """
+ log.info("Triggering Scene inventory tool")
+ item = MainThreadItem(self.tools_helper.show_scene_inventory)
+ # Do not wait for result of callback
+ self._execute_in_main_thread(item, wait=False)
+ return
+
+ async def library_loader_tool(self):
+ log.info("Triggering Library loader tool")
+ item = MainThreadItem(self.tools_helper.show_library_loader)
+ self._execute_in_main_thread(item)
+ return
+
+ async def experimental_tools(self):
+ log.info("Triggering Library loader tool")
+ item = MainThreadItem(self.tools_helper.show_experimental_tools_dialog)
+ self._execute_in_main_thread(item)
+ return
+
+ async def _async_execute_in_main_thread(self, item, **kwargs):
+ await self.communication_obj.async_execute_in_main_thread(
+ item, **kwargs
+ )
+
+ def _execute_in_main_thread(self, item, **kwargs):
+ return self.communication_obj.execute_in_main_thread(item, **kwargs)
+
+
+class MainThreadItem:
+ """Structure to store information about callback in main thread.
+
+ Item should be used to execute callback in main thread which may be needed
+ for execution of Qt objects.
+
+ Item store callback (callable variable), arguments and keyword arguments
+ for the callback. Item hold information about it's process.
+ """
+ not_set = object()
+ sleep_time = 0.1
+
+ def __init__(self, callback, *args, **kwargs):
+ self.done = False
+ self.exception = self.not_set
+ self.result = self.not_set
+ self.callback = callback
+ self.args = args
+ self.kwargs = kwargs
+
+ def execute(self):
+ """Execute callback and store it's result.
+
+ Method must be called from main thread. Item is marked as `done`
+ when callback execution finished. Store output of callback of exception
+ information when callback raise one.
+ """
+ log.debug("Executing process in main thread")
+ if self.done:
+ log.warning("- item is already processed")
+ return
+
+ callback = self.callback
+ args = self.args
+ kwargs = self.kwargs
+ log.info("Running callback: {}".format(str(callback)))
+ try:
+ result = callback(*args, **kwargs)
+ self.result = result
+
+ except Exception as exc:
+ self.exception = exc
+
+ finally:
+ self.done = True
+
+ def wait(self):
+ """Wait for result from main thread.
+
+ This method stops current thread until callback is executed.
+
+ Returns:
+ object: Output of callback. May be any type or object.
+
+ Raises:
+ Exception: Reraise any exception that happened during callback
+ execution.
+ """
+ while not self.done:
+ time.sleep(self.sleep_time)
+
+ if self.exception is self.not_set:
+ return self.result
+ raise self.exception
+
+ async def async_wait(self):
+ """Wait for result from main thread.
+
+ Returns:
+ object: Output of callback. May be any type or object.
+
+ Raises:
+ Exception: Reraise any exception that happened during callback
+ execution.
+ """
+ while not self.done:
+ await asyncio.sleep(self.sleep_time)
+
+ if self.exception is self.not_set:
+ return self.result
+ raise self.exception
+
+
+class BaseCommunicator:
+ def __init__(self):
+ self.process = None
+ self.websocket_server = None
+ self.websocket_rpc = None
+ self.exit_code = None
+ self._connected_client = None
+
+ @property
+ def server_is_running(self):
+ if self.websocket_server is None:
+ return False
+ return self.websocket_server.server_is_running
+
+ def _windows_file_process(self, src_dst_mapping, to_remove):
+ """Windows specific file processing asking for admin permissions.
+
+ It is required to have administration permissions to modify plugin
+ files in TVPaint installation folder.
+
+ Method requires `pywin32` python module.
+
+ Args:
+ src_dst_mapping (list, tuple, set): Mapping of source file to
+ destination. Both must be full path. Each item must be iterable
+ of size 2 `(C:/src/file.dll, C:/dst/file.dll)`.
+ to_remove (list): Fullpath to files that should be removed.
+ """
+
+ import pythoncom
+ from win32comext.shell import shell
+
+ # Create temp folder where plugin files are temporary copied
+ # - reason is that copy to TVPaint requires administartion permissions
+ # but admin may not have access to source folder
+ tmp_dir = os.path.normpath(
+ tempfile.mkdtemp(prefix="tvpaint_copy_")
+ )
+
+ # Copy source to temp folder and create new mapping
+ dst_folders = collections.defaultdict(list)
+ new_src_dst_mapping = []
+ for old_src, dst in src_dst_mapping:
+ new_src = os.path.join(tmp_dir, os.path.split(old_src)[1])
+ shutil.copy(old_src, new_src)
+ new_src_dst_mapping.append((new_src, dst))
+
+ for src, dst in new_src_dst_mapping:
+ src = os.path.normpath(src)
+ dst = os.path.normpath(dst)
+ dst_filename = os.path.basename(dst)
+ dst_folder_path = os.path.dirname(dst)
+ dst_folders[dst_folder_path].append((dst_filename, src))
+
+ # create an instance of IFileOperation
+ fo = pythoncom.CoCreateInstance(
+ shell.CLSID_FileOperation,
+ None,
+ pythoncom.CLSCTX_ALL,
+ shell.IID_IFileOperation
+ )
+ # Add delete command to file operation object
+ for filepath in to_remove:
+ item = shell.SHCreateItemFromParsingName(
+ filepath, None, shell.IID_IShellItem
+ )
+ fo.DeleteItem(item)
+
+ # here you can use SetOperationFlags, progress Sinks, etc.
+ for folder_path, items in dst_folders.items():
+ # create an instance of IShellItem for the target folder
+ folder_item = shell.SHCreateItemFromParsingName(
+ folder_path, None, shell.IID_IShellItem
+ )
+ for _dst_filename, source_file_path in items:
+ # create an instance of IShellItem for the source item
+ copy_item = shell.SHCreateItemFromParsingName(
+ source_file_path, None, shell.IID_IShellItem
+ )
+ # queue the copy operation
+ fo.CopyItem(copy_item, folder_item, _dst_filename, None)
+
+ # commit
+ fo.PerformOperations()
+
+ # Remove temp folder
+ shutil.rmtree(tmp_dir)
+
+ def _prepare_windows_plugin(self, launch_args):
+ """Copy plugin to TVPaint plugins and set PATH to dependencies.
+
+ Check if plugin in TVPaint's plugins exist and match to plugin
+ version to current implementation version. Based on 64-bit or 32-bit
+ version of the plugin. Path to libraries required for plugin is added
+ to PATH variable.
+ """
+
+ host_executable = launch_args[0]
+ executable_file = os.path.basename(host_executable)
+ if "64bit" in executable_file:
+ subfolder = "windows_x64"
+ elif "32bit" in executable_file:
+ subfolder = "windows_x86"
+ else:
+ raise ValueError(
+ "Can't determine if executable "
+ "leads to 32-bit or 64-bit TVPaint!"
+ )
+
+ plugin_files_path = get_plugin_files_path()
+ # Folder for right windows plugin files
+ source_plugins_dir = os.path.join(plugin_files_path, subfolder)
+
+ # Path to libraries (.dll) required for plugin library
+ # - additional libraries can be copied to TVPaint installation folder
+ # (next to executable) or added to PATH environment variable
+ additional_libs_folder = os.path.join(
+ source_plugins_dir,
+ "additional_libraries"
+ )
+ additional_libs_folder = additional_libs_folder.replace("\\", "/")
+ if (
+ os.path.exists(additional_libs_folder)
+ and additional_libs_folder not in os.environ["PATH"]
+ ):
+ os.environ["PATH"] += (os.pathsep + additional_libs_folder)
+
+ # Path to TVPaint's plugins folder (where we want to add our plugin)
+ host_plugins_path = os.path.join(
+ os.path.dirname(host_executable),
+ "plugins"
+ )
+
+ # Files that must be copied to TVPaint's plugin folder
+ plugin_dir = os.path.join(source_plugins_dir, "plugin")
+
+ to_copy = []
+ to_remove = []
+ # Remove old plugin name
+ deprecated_filepath = os.path.join(
+ host_plugins_path, "AvalonPlugin.dll"
+ )
+ if os.path.exists(deprecated_filepath):
+ to_remove.append(deprecated_filepath)
+
+ for filename in os.listdir(plugin_dir):
+ src_full_path = os.path.join(plugin_dir, filename)
+ dst_full_path = os.path.join(host_plugins_path, filename)
+ if dst_full_path in to_remove:
+ to_remove.remove(dst_full_path)
+
+ if (
+ not os.path.exists(dst_full_path)
+ or not filecmp.cmp(src_full_path, dst_full_path)
+ ):
+ to_copy.append((src_full_path, dst_full_path))
+
+ # Skip copy if everything is done
+ if not to_copy and not to_remove:
+ return
+
+ # Try to copy
+ try:
+ self._windows_file_process(to_copy, to_remove)
+ except Exception:
+ log.error("Plugin copy failed", exc_info=True)
+
+ # Validate copy was done
+ invalid_copy = []
+ for src, dst in to_copy:
+ if not os.path.exists(dst) or not filecmp.cmp(src, dst):
+ invalid_copy.append((src, dst))
+
+ # Validate delete was dones
+ invalid_remove = []
+ for filepath in to_remove:
+ if os.path.exists(filepath):
+ invalid_remove.append(filepath)
+
+ if not invalid_remove and not invalid_copy:
+ return
+
+ msg_parts = []
+ if invalid_remove:
+ msg_parts.append(
+ "Failed to remove files: {}".format(", ".join(invalid_remove))
+ )
+
+ if invalid_copy:
+ _invalid = [
+ "\"{}\" -> \"{}\"".format(src, dst)
+ for src, dst in invalid_copy
+ ]
+ msg_parts.append(
+ "Failed to copy files: {}".format(", ".join(_invalid))
+ )
+ raise RuntimeError(" & ".join(msg_parts))
+
+ def _launch_tv_paint(self, launch_args):
+ flags = (
+ subprocess.DETACHED_PROCESS
+ | subprocess.CREATE_NEW_PROCESS_GROUP
+ )
+ env = os.environ.copy()
+ # Remove QuickTime from PATH on windows
+ # - quicktime overrides TVPaint's ffmpeg encode/decode which may
+ # cause issues on loading
+ if platform.system().lower() == "windows":
+ new_path = []
+ for path in env["PATH"].split(os.pathsep):
+ if path and "quicktime" not in path.lower():
+ new_path.append(path)
+ env["PATH"] = os.pathsep.join(new_path)
+
+ kwargs = {
+ "env": env,
+ "creationflags": flags
+ }
+ self.process = subprocess.Popen(launch_args, **kwargs)
+
+ def _create_routes(self):
+ self.websocket_rpc = BaseTVPaintRpc(
+ self, loop=self.websocket_server.loop
+ )
+ self.websocket_server.add_route(
+ "*", "/", self.websocket_rpc.handle_request
+ )
+
+ def _start_webserver(self):
+ self.websocket_server.start()
+ # Make sure RPC is using same loop as websocket server
+ while not self.websocket_server.server_is_running:
+ time.sleep(0.1)
+
+ def _stop_webserver(self):
+ self.websocket_server.stop()
+
+ def _exit(self, exit_code=None):
+ self._stop_webserver()
+ if exit_code is not None:
+ self.exit_code = exit_code
+
+ if self.exit_code is None:
+ self.exit_code = 0
+
+ def stop(self):
+ """Stop communication and currently running python process."""
+ log.info("Stopping communication")
+ self._exit()
+
+ def launch(self, launch_args):
+ """Prepare all required data and launch host.
+
+ First is prepared websocket server as communication point for host,
+ when server is ready to use host is launched as subprocess.
+ """
+ if platform.system().lower() == "windows":
+ self._prepare_windows_plugin(launch_args)
+
+ # Launch TVPaint and the websocket server.
+ log.info("Launching TVPaint")
+ self.websocket_server = WebSocketServer()
+
+ self._create_routes()
+
+ os.environ["WEBSOCKET_URL"] = "ws://localhost:{}".format(
+ self.websocket_server.port
+ )
+
+ log.info("Added request handler for url: {}".format(
+ os.environ["WEBSOCKET_URL"]
+ ))
+
+ self._start_webserver()
+
+ # Start TVPaint when server is running
+ self._launch_tv_paint(launch_args)
+
+ log.info("Waiting for client connection")
+ while True:
+ if self.process.poll() is not None:
+ log.debug("Host process is not alive. Exiting")
+ self._exit(1)
+ return
+
+ if self.websocket_rpc.client_connected():
+ log.info("Client has connected")
+ break
+ time.sleep(0.5)
+
+ self._on_client_connect()
+
+ emit_event("application.launched")
+
+ def _on_client_connect(self):
+ self._initial_textfile_write()
+
+ def _initial_textfile_write(self):
+ """Show popup about Write to file at start of TVPaint."""
+ tmp_file = tempfile.NamedTemporaryFile(
+ mode="w", prefix="a_tvp_", suffix=".txt", delete=False
+ )
+ tmp_file.close()
+ tmp_filepath = tmp_file.name.replace("\\", "/")
+ george_script = (
+ "tv_writetextfile \"strict\" \"append\" \"{}\" \"empty\""
+ ).format(tmp_filepath)
+
+ result = CommunicationWrapper.execute_george(george_script)
+
+ # Remote the file
+ os.remove(tmp_filepath)
+
+ if result is None:
+ log.warning(
+ "Host was probably closed before plugin was initialized."
+ )
+ elif result.lower() == "forbidden":
+ log.warning("User didn't confirm saving files.")
+
+ def _client(self):
+ if not self.websocket_rpc:
+ log.warning("Communicator's server did not start yet.")
+ return None
+
+ for client in self.websocket_rpc.clients:
+ if not client.ws.closed:
+ return client
+ log.warning("Client is not yet connected to Communicator.")
+ return None
+
+ def client(self):
+ if not self._connected_client or self._connected_client.ws.closed:
+ self._connected_client = self._client()
+ return self._connected_client
+
+ def send_request(self, method, params=None):
+ client = self.client()
+ if not client:
+ return
+
+ return self.websocket_rpc.send_request(
+ client, method, params
+ )
+
+ def send_notification(self, method, params=None):
+ client = self.client()
+ if not client:
+ return
+
+ self.websocket_rpc.send_notification(
+ client, method, params
+ )
+
+ def execute_george(self, george_script):
+ """Execute passed goerge script in TVPaint."""
+ return self.send_request(
+ "execute_george", [george_script]
+ )
+
+ def execute_george_through_file(self, george_script):
+ """Execute george script with temp file.
+
+ Allows to execute multiline george script without stopping websocket
+ client.
+
+ On windows make sure script does not contain paths with backwards
+ slashes in paths, TVPaint won't execute properly in that case.
+
+ Args:
+ george_script (str): George script to execute. May be multilined.
+ """
+ temporary_file = tempfile.NamedTemporaryFile(
+ mode="w", prefix="a_tvp_", suffix=".grg", delete=False
+ )
+ temporary_file.write(george_script)
+ temporary_file.close()
+ temp_file_path = temporary_file.name.replace("\\", "/")
+ self.execute_george("tv_runscript {}".format(temp_file_path))
+ os.remove(temp_file_path)
+
+
+class QtCommunicator(BaseCommunicator):
+ menu_definitions = {
+ "title": "OpenPype Tools",
+ "menu_items": [
+ {
+ "callback": "workfiles_tool",
+ "label": "Workfiles",
+ "help": "Open workfiles tool"
+ }, {
+ "callback": "loader_tool",
+ "label": "Load",
+ "help": "Open loader tool"
+ }, {
+ "callback": "creator_tool",
+ "label": "Create",
+ "help": "Open creator tool"
+ }, {
+ "callback": "scene_inventory_tool",
+ "label": "Scene inventory",
+ "help": "Open scene inventory tool"
+ }, {
+ "callback": "publish_tool",
+ "label": "Publish",
+ "help": "Open publisher"
+ }, {
+ "callback": "library_loader_tool",
+ "label": "Library",
+ "help": "Open library loader tool"
+ }, {
+ "callback": "subset_manager_tool",
+ "label": "Subset Manager",
+ "help": "Open subset manager tool"
+ }, {
+ "callback": "experimental_tools",
+ "label": "Experimental tools",
+ "help": "Open experimental tools dialog"
+ }
+ ]
+ }
+
+ def __init__(self, qt_app):
+ super().__init__()
+ self.callback_queue = Queue()
+ self.qt_app = qt_app
+
+ def _create_routes(self):
+ self.websocket_rpc = QtTVPaintRpc(
+ self, loop=self.websocket_server.loop
+ )
+ self.websocket_server.add_route(
+ "*", "/", self.websocket_rpc.handle_request
+ )
+
+ def execute_in_main_thread(self, main_thread_item, wait=True):
+ """Add `MainThreadItem` to callback queue and wait for result."""
+ self.callback_queue.put(main_thread_item)
+ if wait:
+ return main_thread_item.wait()
+ return
+
+ async def async_execute_in_main_thread(self, main_thread_item, wait=True):
+ """Add `MainThreadItem` to callback queue and wait for result."""
+ self.callback_queue.put(main_thread_item)
+ if wait:
+ return await main_thread_item.async_wait()
+
+ def main_thread_listen(self):
+ """Get last `MainThreadItem` from queue.
+
+ Must be called from main thread.
+
+ Method checks if host process is still running as it may cause
+ issues if not.
+ """
+ # check if host still running
+ if self.process.poll() is not None:
+ self._exit()
+ return None
+
+ if self.callback_queue.empty():
+ return None
+ return self.callback_queue.get()
+
+ def _on_client_connect(self):
+ super()._on_client_connect()
+ self._build_menu()
+
+ def _build_menu(self):
+ self.send_request(
+ "define_menu", [self.menu_definitions]
+ )
+
+ def _exit(self, *args, **kwargs):
+ super()._exit(*args, **kwargs)
+ emit_event("application.exit")
+ self.qt_app.exit(self.exit_code)
diff --git a/openpype/hosts/tvpaint/api/launch_script.py b/openpype/hosts/tvpaint/api/launch_script.py
new file mode 100644
index 00000000000..0b25027fc63
--- /dev/null
+++ b/openpype/hosts/tvpaint/api/launch_script.py
@@ -0,0 +1,84 @@
+import os
+import sys
+import signal
+import traceback
+import ctypes
+import platform
+import logging
+
+from Qt import QtWidgets, QtCore, QtGui
+
+from openpype import style
+from openpype.pipeline import install_host
+from openpype.hosts.tvpaint.api.communication_server import (
+ CommunicationWrapper
+)
+from openpype.hosts.tvpaint import api as tvpaint_host
+
+log = logging.getLogger(__name__)
+
+
+def safe_excepthook(*args):
+ traceback.print_exception(*args)
+
+
+def main(launch_args):
+ # Be sure server won't crash at any moment but just print traceback
+ sys.excepthook = safe_excepthook
+
+ # Create QtApplication for tools
+ # - QApplicaiton is also main thread/event loop of the server
+ qt_app = QtWidgets.QApplication([])
+
+ # Execute pipeline installation
+ install_host(tvpaint_host)
+
+ # Create Communicator object and trigger launch
+ # - this must be done before anything is processed
+ communicator = CommunicationWrapper.create_qt_communicator(qt_app)
+ communicator.launch(launch_args)
+
+ def process_in_main_thread():
+ """Execution of `MainThreadItem`."""
+ item = communicator.main_thread_listen()
+ if item:
+ item.execute()
+
+ timer = QtCore.QTimer()
+ timer.setInterval(100)
+ timer.timeout.connect(process_in_main_thread)
+ timer.start()
+
+ # Register terminal signal handler
+ def signal_handler(*_args):
+ print("You pressed Ctrl+C. Process ended.")
+ communicator.stop()
+
+ signal.signal(signal.SIGINT, signal_handler)
+ signal.signal(signal.SIGTERM, signal_handler)
+
+ qt_app.setQuitOnLastWindowClosed(False)
+ qt_app.setStyleSheet(style.load_stylesheet())
+
+ # Load avalon icon
+ icon_path = style.app_icon_path()
+ if icon_path:
+ icon = QtGui.QIcon(icon_path)
+ qt_app.setWindowIcon(icon)
+
+ # Set application name to be able show application icon in task bar
+ if platform.system().lower() == "windows":
+ ctypes.windll.shell32.SetCurrentProcessExplicitAppUserModelID(
+ u"WebsocketServer"
+ )
+
+ # Run Qt application event processing
+ sys.exit(qt_app.exec_())
+
+
+if __name__ == "__main__":
+ args = list(sys.argv)
+ if os.path.abspath(__file__) == os.path.normpath(args[0]):
+ # Pop path to script
+ args.pop(0)
+ main(args)
diff --git a/openpype/hosts/tvpaint/api/lib.py b/openpype/hosts/tvpaint/api/lib.py
index 539cebe6468..a341f488592 100644
--- a/openpype/hosts/tvpaint/api/lib.py
+++ b/openpype/hosts/tvpaint/api/lib.py
@@ -1,85 +1,537 @@
-from PIL import Image
+import os
+import logging
+import tempfile
-import avalon.io
-from avalon.tvpaint.lib import execute_george
+from . import CommunicationWrapper
+log = logging.getLogger(__name__)
-def composite_images(input_image_paths, output_filepath):
- """Composite images in order from passed list.
- Raises:
- ValueError: When entered list is empty.
- """
- if not input_image_paths:
- raise ValueError("Nothing to composite.")
+def execute_george(george_script, communicator=None):
+ if not communicator:
+ communicator = CommunicationWrapper.communicator
+ return communicator.execute_george(george_script)
+
- img_obj = None
- for image_filepath in input_image_paths:
- _img_obj = Image.open(image_filepath)
- if img_obj is None:
- img_obj = _img_obj
- else:
- img_obj.alpha_composite(_img_obj)
- img_obj.save(output_filepath)
+def execute_george_through_file(george_script, communicator=None):
+ """Execute george script with temp file.
+ Allows to execute multiline george script without stopping websocket
+ client.
-def set_context_settings(asset_doc=None):
- """Set workfile settings by asset document data.
+ On windows make sure script does not contain paths with backwards
+ slashes in paths, TVPaint won't execute properly in that case.
- Change fps, resolution and frame start/end.
+ Args:
+ george_script (str): George script to execute. May be multilined.
"""
- if asset_doc is None:
- # Use current session asset if not passed
- asset_doc = avalon.io.find_one({
- "type": "asset",
- "name": avalon.io.Session["AVALON_ASSET"]
- })
-
- project_doc = avalon.io.find_one({"type": "project"})
-
- framerate = asset_doc["data"].get("fps")
- if framerate is None:
- framerate = project_doc["data"].get("fps")
-
- if framerate is not None:
- execute_george(
- "tv_framerate {} \"timestretch\"".format(framerate)
- )
- else:
- print("Framerate was not found!")
+ if not communicator:
+ communicator = CommunicationWrapper.communicator
+
+ return communicator.execute_george_through_file(george_script)
+
- width_key = "resolutionWidth"
- height_key = "resolutionHeight"
+def parse_layers_data(data):
+ """Parse layers data loaded in 'get_layers_data'."""
+ layers = []
+ layers_raw = data.split("\n")
+ for layer_raw in layers_raw:
+ layer_raw = layer_raw.strip()
+ if not layer_raw:
+ continue
+ (
+ layer_id, group_id, visible, position, opacity, name,
+ layer_type,
+ frame_start, frame_end, prelighttable, postlighttable,
+ selected, editable, sencil_state
+ ) = layer_raw.split("|")
+ layer = {
+ "layer_id": int(layer_id),
+ "group_id": int(group_id),
+ "visible": visible == "ON",
+ "position": int(position),
+ "opacity": int(opacity),
+ "name": name,
+ "type": layer_type,
+ "frame_start": int(frame_start),
+ "frame_end": int(frame_end),
+ "prelighttable": prelighttable == "1",
+ "postlighttable": postlighttable == "1",
+ "selected": selected == "1",
+ "editable": editable == "1",
+ "sencil_state": sencil_state
+ }
+ layers.append(layer)
+ return layers
- width = asset_doc["data"].get(width_key)
- height = asset_doc["data"].get(height_key)
- if width is None or height is None:
- width = project_doc["data"].get(width_key)
- height = project_doc["data"].get(height_key)
- if width is None or height is None:
- print("Resolution was not found!")
+def get_layers_data_george_script(output_filepath, layer_ids=None):
+ """Prepare george script which will collect all layers from workfile."""
+ output_filepath = output_filepath.replace("\\", "/")
+ george_script_lines = [
+ # Variable containing full path to output file
+ "output_path = \"{}\"".format(output_filepath),
+ # Get Current Layer ID
+ "tv_LayerCurrentID",
+ "current_layer_id = result"
+ ]
+ # Script part for getting and storing layer information to temp
+ layer_data_getter = (
+ # Get information about layer's group
+ "tv_layercolor \"get\" layer_id",
+ "group_id = result",
+ "tv_LayerInfo layer_id",
+ (
+ "PARSE result visible position opacity name"
+ " type startFrame endFrame prelighttable postlighttable"
+ " selected editable sencilState"
+ ),
+ # Check if layer ID match `tv_LayerCurrentID`
+ "IF CMP(current_layer_id, layer_id)==1",
+ # - mark layer as selected if layer id match to current layer id
+ "selected=1",
+ "END",
+ # Prepare line with data separated by "|"
+ (
+ "line = layer_id'|'group_id'|'visible'|'position'|'opacity'|'"
+ "name'|'type'|'startFrame'|'endFrame'|'prelighttable'|'"
+ "postlighttable'|'selected'|'editable'|'sencilState"
+ ),
+ # Write data to output file
+ "tv_writetextfile \"strict\" \"append\" '\"'output_path'\"' line",
+ )
+
+ # Collect data for all layers if layers are not specified
+ if layer_ids is None:
+ george_script_lines.extend((
+ # Layer loop variables
+ "loop = 1",
+ "idx = 0",
+ # Layers loop
+ "WHILE loop",
+ "tv_LayerGetID idx",
+ "layer_id = result",
+ "idx = idx + 1",
+ # Stop loop if layer_id is "NONE"
+ "IF CMP(layer_id, \"NONE\")==1",
+ "loop = 0",
+ "ELSE",
+ *layer_data_getter,
+ "END",
+ "END"
+ ))
else:
- execute_george("tv_resizepage {} {} 0".format(width, height))
+ for layer_id in layer_ids:
+ george_script_lines.append("layer_id = {}".format(layer_id))
+ george_script_lines.extend(layer_data_getter)
+
+ return "\n".join(george_script_lines)
+
+
+def layers_data(layer_ids=None, communicator=None):
+ """Backwards compatible function of 'get_layers_data'."""
+ return get_layers_data(layer_ids, communicator)
+
+
+def get_layers_data(layer_ids=None, communicator=None):
+ """Collect all layers information from currently opened workfile."""
+ output_file = tempfile.NamedTemporaryFile(
+ mode="w", prefix="a_tvp_", suffix=".txt", delete=False
+ )
+ output_file.close()
+ if layer_ids is not None and isinstance(layer_ids, int):
+ layer_ids = [layer_ids]
+
+ output_filepath = output_file.name
+
+ george_script = get_layers_data_george_script(output_filepath, layer_ids)
+
+ execute_george_through_file(george_script, communicator)
+
+ with open(output_filepath, "r") as stream:
+ data = stream.read()
+
+ output = parse_layers_data(data)
+ os.remove(output_filepath)
+ return output
+
+
+def parse_group_data(data):
+ """Parse group data collected in 'get_groups_data'."""
+ output = []
+ groups_raw = data.split("\n")
+ for group_raw in groups_raw:
+ group_raw = group_raw.strip()
+ if not group_raw:
+ continue
+
+ parts = group_raw.split("|")
+ # Check for length and concatenate 2 last items until length match
+ # - this happens if name contain spaces
+ while len(parts) > 6:
+ last_item = parts.pop(-1)
+ parts[-1] = "|".join([parts[-1], last_item])
+ clip_id, group_id, red, green, blue, name = parts
+
+ group = {
+ "group_id": int(group_id),
+ "name": name,
+ "clip_id": int(clip_id),
+ "red": int(red),
+ "green": int(green),
+ "blue": int(blue),
+ }
+ output.append(group)
+ return output
+
+
+def groups_data(communicator=None):
+ """Backwards compatible function of 'get_groups_data'."""
+ return get_groups_data(communicator)
+
+
+def get_groups_data(communicator=None):
+ """Information about groups from current workfile."""
+ output_file = tempfile.NamedTemporaryFile(
+ mode="w", prefix="a_tvp_", suffix=".txt", delete=False
+ )
+ output_file.close()
+
+ output_filepath = output_file.name.replace("\\", "/")
+ george_script_lines = (
+ # Variable containing full path to output file
+ "output_path = \"{}\"".format(output_filepath),
+ "empty = 0",
+ # Loop over 100 groups
+ "FOR idx = 1 TO 100",
+ # Receive information about groups
+ "tv_layercolor \"getcolor\" 0 idx",
+ "PARSE result clip_id group_index c_red c_green c_blue group_name",
+ # Create and add line to output file
+ "line = clip_id'|'group_index'|'c_red'|'c_green'|'c_blue'|'group_name",
+ "tv_writetextfile \"strict\" \"append\" '\"'output_path'\"' line",
+ "END",
+ )
+ george_script = "\n".join(george_script_lines)
+ execute_george_through_file(george_script, communicator)
+
+ with open(output_filepath, "r") as stream:
+ data = stream.read()
+
+ output = parse_group_data(data)
+ os.remove(output_filepath)
+ return output
+
+
+def get_layers_pre_post_behavior(layer_ids, communicator=None):
+ """Collect data about pre and post behavior of layer ids.
+
+ Pre and Post behaviors is enumerator of possible values:
+ - "none"
+ - "repeat" / "loop"
+ - "pingpong"
+ - "hold"
+
+ Example output:
+ ```json
+ {
+ 0: {
+ "pre": "none",
+ "post": "loop"
+ }
+ }
+ ```
+
+ Returns:
+ dict: Key is layer id value is dictionary with "pre" and "post" keys.
+ """
+ # Skip if is empty
+ if not layer_ids:
+ return {}
+
+ # Auto convert to list
+ if not isinstance(layer_ids, (list, set, tuple)):
+ layer_ids = [layer_ids]
+
+ # Prepare temp file
+ output_file = tempfile.NamedTemporaryFile(
+ mode="w", prefix="a_tvp_", suffix=".txt", delete=False
+ )
+ output_file.close()
+
+ output_filepath = output_file.name.replace("\\", "/")
+ george_script_lines = [
+ # Variable containing full path to output file
+ "output_path = \"{}\"".format(output_filepath),
+ ]
+ for layer_id in layer_ids:
+ george_script_lines.extend([
+ "layer_id = {}".format(layer_id),
+ "tv_layerprebehavior layer_id",
+ "pre_beh = result",
+ "tv_layerpostbehavior layer_id",
+ "post_beh = result",
+ "line = layer_id'|'pre_beh'|'post_beh",
+ "tv_writetextfile \"strict\" \"append\" '\"'output_path'\"' line"
+ ])
+
+ george_script = "\n".join(george_script_lines)
+ execute_george_through_file(george_script, communicator)
+
+ # Read data
+ with open(output_filepath, "r") as stream:
+ data = stream.read()
- frame_start = asset_doc["data"].get("frameStart")
- frame_end = asset_doc["data"].get("frameEnd")
+ # Remove temp file
+ os.remove(output_filepath)
- if frame_start is None or frame_end is None:
- print("Frame range was not found!")
- return
+ # Parse data
+ output = {}
+ raw_lines = data.split("\n")
+ for raw_line in raw_lines:
+ line = raw_line.strip()
+ if not line:
+ continue
+ parts = line.split("|")
+ if len(parts) != 3:
+ continue
+ layer_id, pre_beh, post_beh = parts
+ output[int(layer_id)] = {
+ "pre": pre_beh.lower(),
+ "post": post_beh.lower()
+ }
+ return output
- handles = asset_doc["data"].get("handles") or 0
- handle_start = asset_doc["data"].get("handleStart")
- handle_end = asset_doc["data"].get("handleEnd")
- if handle_start is None or handle_end is None:
- handle_start = handles
- handle_end = handles
+def get_layers_exposure_frames(layer_ids, layers_data=None, communicator=None):
+ """Get exposure frames.
+
+ Easily said returns frames where keyframes are. Recognized with george
+ function `tv_exposureinfo` returning "Head".
+
+ Args:
+ layer_ids (list): Ids of a layers for which exposure frames should
+ look for.
+ layers_data (list): Precollected layers data. If are not passed then
+ 'get_layers_data' is used.
+ communicator (BaseCommunicator): Communicator used for communication
+ with TVPaint.
+
+ Returns:
+ dict: Frames where exposure is set to "Head" by layer id.
+ """
+
+ if layers_data is None:
+ layers_data = get_layers_data(layer_ids)
+ _layers_by_id = {
+ layer["layer_id"]: layer
+ for layer in layers_data
+ }
+ layers_by_id = {
+ layer_id: _layers_by_id.get(layer_id)
+ for layer_id in layer_ids
+ }
+ tmp_file = tempfile.NamedTemporaryFile(
+ mode="w", prefix="a_tvp_", suffix=".txt", delete=False
+ )
+ tmp_file.close()
+ tmp_output_path = tmp_file.name.replace("\\", "/")
+ george_script_lines = [
+ "output_path = \"{}\"".format(tmp_output_path)
+ ]
+
+ output = {}
+ layer_id_mapping = {}
+ for layer_id, layer_data in layers_by_id.items():
+ layer_id_mapping[str(layer_id)] = layer_id
+ output[layer_id] = []
+ if not layer_data:
+ continue
+ first_frame = layer_data["frame_start"]
+ last_frame = layer_data["frame_end"]
+ george_script_lines.extend([
+ "line = \"\"",
+ "layer_id = {}".format(layer_id),
+ "line = line''layer_id",
+ "tv_layerset layer_id",
+ "frame = {}".format(first_frame),
+ "WHILE (frame <= {})".format(last_frame),
+ "tv_exposureinfo frame",
+ "exposure = result",
+ "IF (CMP(exposure, \"Head\") == 1)",
+ "line = line'|'frame",
+ "END",
+ "frame = frame + 1",
+ "END",
+ "tv_writetextfile \"strict\" \"append\" '\"'output_path'\"' line"
+ ])
+
+ execute_george_through_file("\n".join(george_script_lines), communicator)
+
+ with open(tmp_output_path, "r") as stream:
+ data = stream.read()
+
+ os.remove(tmp_output_path)
+
+ lines = []
+ for line in data.split("\n"):
+ line = line.strip()
+ if line:
+ lines.append(line)
+
+ for line in lines:
+ line_items = list(line.split("|"))
+ layer_id = line_items.pop(0)
+ _layer_id = layer_id_mapping[layer_id]
+ output[_layer_id] = [int(frame) for frame in line_items]
+
+ return output
+
+
+def get_exposure_frames(
+ layer_id, first_frame=None, last_frame=None, communicator=None
+):
+ """Get exposure frames.
+
+ Easily said returns frames where keyframes are. Recognized with george
+ function `tv_exposureinfo` returning "Head".
+
+ Args:
+ layer_id (int): Id of a layer for which exposure frames should
+ look for.
+ first_frame (int): From which frame will look for exposure frames.
+ Used layers first frame if not entered.
+ last_frame (int): Last frame where will look for exposure frames.
+ Used layers last frame if not entered.
+
+ Returns:
+ list: Frames where exposure is set to "Head".
+ """
+ if first_frame is None or last_frame is None:
+ layer = layers_data(layer_id)[0]
+ if first_frame is None:
+ first_frame = layer["frame_start"]
+ if last_frame is None:
+ last_frame = layer["frame_end"]
+
+ tmp_file = tempfile.NamedTemporaryFile(
+ mode="w", prefix="a_tvp_", suffix=".txt", delete=False
+ )
+ tmp_file.close()
+ tmp_output_path = tmp_file.name.replace("\\", "/")
+ george_script_lines = [
+ "tv_layerset {}".format(layer_id),
+ "output_path = \"{}\"".format(tmp_output_path),
+ "output = \"\"",
+ "frame = {}".format(first_frame),
+ "WHILE (frame <= {})".format(last_frame),
+ "tv_exposureinfo frame",
+ "exposure = result",
+ "IF (CMP(exposure, \"Head\") == 1)",
+ "IF (CMP(output, \"\") == 1)",
+ "output = output''frame",
+ "ELSE",
+ "output = output'|'frame",
+ "END",
+ "END",
+ "frame = frame + 1",
+ "END",
+ "tv_writetextfile \"strict\" \"append\" '\"'output_path'\"' output"
+ ]
+
+ execute_george_through_file("\n".join(george_script_lines), communicator)
+
+ with open(tmp_output_path, "r") as stream:
+ data = stream.read()
+
+ os.remove(tmp_output_path)
+
+ lines = []
+ for line in data.split("\n"):
+ line = line.strip()
+ if line:
+ lines.append(line)
+
+ exposure_frames = []
+ for line in lines:
+ for frame in line.split("|"):
+ exposure_frames.append(int(frame))
+ return exposure_frames
+
+
+def get_scene_data(communicator=None):
+ """Scene data of currently opened scene.
+
+ Result contains resolution, pixel aspect, fps mark in/out with states,
+ frame start and background color.
+
+ Returns:
+ dict: Scene data collected in many ways.
+ """
+ workfile_info = execute_george("tv_projectinfo", communicator)
+ workfile_info_parts = workfile_info.split(" ")
+
+ # Project frame start - not used
+ workfile_info_parts.pop(-1)
+ field_order = workfile_info_parts.pop(-1)
+ frame_rate = float(workfile_info_parts.pop(-1))
+ pixel_apsect = float(workfile_info_parts.pop(-1))
+ height = int(workfile_info_parts.pop(-1))
+ width = int(workfile_info_parts.pop(-1))
+
+ # Marks return as "{frame - 1} {state} ", example "0 set".
+ result = execute_george("tv_markin", communicator)
+ mark_in_frame, mark_in_state, _ = result.split(" ")
+
+ result = execute_george("tv_markout", communicator)
+ mark_out_frame, mark_out_state, _ = result.split(" ")
+
+ start_frame = execute_george("tv_startframe", communicator)
+ return {
+ "width": width,
+ "height": height,
+ "pixel_aspect": pixel_apsect,
+ "fps": frame_rate,
+ "field_order": field_order,
+ "mark_in": int(mark_in_frame),
+ "mark_in_state": mark_in_state,
+ "mark_in_set": mark_in_state == "set",
+ "mark_out": int(mark_out_frame),
+ "mark_out_state": mark_out_state,
+ "mark_out_set": mark_out_state == "set",
+ "start_frame": int(start_frame),
+ "bg_color": get_scene_bg_color(communicator)
+ }
+
+
+def get_scene_bg_color(communicator=None):
+ """Background color set on scene.
+
+ Is important for review exporting where scene bg color is used as
+ background.
+ """
+ output_file = tempfile.NamedTemporaryFile(
+ mode="w", prefix="a_tvp_", suffix=".txt", delete=False
+ )
+ output_file.close()
+ output_filepath = output_file.name.replace("\\", "/")
+ george_script_lines = [
+ # Variable containing full path to output file
+ "output_path = \"{}\"".format(output_filepath),
+ "tv_background",
+ "bg_color = result",
+ # Write data to output file
+ "tv_writetextfile \"strict\" \"append\" '\"'output_path'\"' bg_color"
+ ]
+
+ george_script = "\n".join(george_script_lines)
+ execute_george_through_file(george_script, communicator)
- # Always start from 0 Mark In and set only Mark Out
- mark_in = 0
- mark_out = mark_in + (frame_end - frame_start) + handle_start + handle_end
+ with open(output_filepath, "r") as stream:
+ data = stream.read()
- execute_george("tv_markin {} set".format(mark_in))
- execute_george("tv_markout {} set".format(mark_out))
+ os.remove(output_filepath)
+ data = data.strip()
+ if not data:
+ return None
+ return data.split(" ")
diff --git a/openpype/hosts/tvpaint/api/pipeline.py b/openpype/hosts/tvpaint/api/pipeline.py
new file mode 100644
index 00000000000..427c9272648
--- /dev/null
+++ b/openpype/hosts/tvpaint/api/pipeline.py
@@ -0,0 +1,487 @@
+import os
+import json
+import contextlib
+import tempfile
+import logging
+
+import requests
+
+import pyblish.api
+
+from openpype.client import get_project, get_asset_by_name
+from openpype.hosts import tvpaint
+from openpype.api import get_current_project_settings
+from openpype.lib import register_event_callback
+from openpype.pipeline import (
+ legacy_io,
+ register_loader_plugin_path,
+ register_creator_plugin_path,
+ AVALON_CONTAINER_ID,
+)
+
+from .lib import (
+ execute_george,
+ execute_george_through_file
+)
+
+log = logging.getLogger(__name__)
+
+HOST_DIR = os.path.dirname(os.path.abspath(tvpaint.__file__))
+PLUGINS_DIR = os.path.join(HOST_DIR, "plugins")
+PUBLISH_PATH = os.path.join(PLUGINS_DIR, "publish")
+LOAD_PATH = os.path.join(PLUGINS_DIR, "load")
+CREATE_PATH = os.path.join(PLUGINS_DIR, "create")
+
+METADATA_SECTION = "avalon"
+SECTION_NAME_CONTEXT = "context"
+SECTION_NAME_INSTANCES = "instances"
+SECTION_NAME_CONTAINERS = "containers"
+# Maximum length of metadata chunk string
+# TODO find out the max (500 is safe enough)
+TVPAINT_CHUNK_LENGTH = 500
+
+"""TVPaint's Metadata
+
+Metadata are stored to TVPaint's workfile.
+
+Workfile works similar to .ini file but has few limitation. Most important
+limitation is that value under key has limited length. Due to this limitation
+each metadata section/key stores number of "subkeys" that are related to
+the section.
+
+Example:
+Metadata key `"instances"` may have stored value "2". In that case it is
+expected that there are also keys `["instances0", "instances1"]`.
+
+Workfile data looks like:
+```
+[avalon]
+instances0=[{{__dq__}id{__dq__}: {__dq__}pyblish.avalon.instance{__dq__...
+instances1=...more data...
+instances=2
+```
+"""
+
+
+def install():
+ """Install TVPaint-specific functionality."""
+
+ log.info("OpenPype - Installing TVPaint integration")
+ legacy_io.install()
+
+ # Create workdir folder if does not exist yet
+ workdir = legacy_io.Session["AVALON_WORKDIR"]
+ if not os.path.exists(workdir):
+ os.makedirs(workdir)
+
+ pyblish.api.register_host("tvpaint")
+ pyblish.api.register_plugin_path(PUBLISH_PATH)
+ register_loader_plugin_path(LOAD_PATH)
+ register_creator_plugin_path(CREATE_PATH)
+
+ registered_callbacks = (
+ pyblish.api.registered_callbacks().get("instanceToggled") or []
+ )
+ if on_instance_toggle not in registered_callbacks:
+ pyblish.api.register_callback("instanceToggled", on_instance_toggle)
+
+ register_event_callback("application.launched", initial_launch)
+ register_event_callback("application.exit", application_exit)
+
+
+def containerise(
+ name, namespace, members, context, loader, current_containers=None
+):
+ """Add new container to metadata.
+
+ Args:
+ name (str): Container name.
+ namespace (str): Container namespace.
+ members (list): List of members that were loaded and belongs
+ to the container (layer names).
+ current_containers (list): Preloaded containers. Should be used only
+ on update/switch when containers were modified during the process.
+
+ Returns:
+ dict: Container data stored to workfile metadata.
+ """
+
+ container_data = {
+ "schema": "openpype:container-2.0",
+ "id": AVALON_CONTAINER_ID,
+ "members": members,
+ "name": name,
+ "namespace": namespace,
+ "loader": str(loader),
+ "representation": str(context["representation"]["_id"])
+ }
+ if current_containers is None:
+ current_containers = ls()
+
+ # Add container to containers list
+ current_containers.append(container_data)
+
+ # Store data to metadata
+ write_workfile_metadata(SECTION_NAME_CONTAINERS, current_containers)
+
+ return container_data
+
+
+@contextlib.contextmanager
+def maintained_selection():
+ # TODO implement logic
+ try:
+ yield
+ finally:
+ pass
+
+
+def split_metadata_string(text, chunk_length=None):
+ """Split string by length.
+
+ Split text to chunks by entered length.
+ Example:
+ ```python
+ text = "ABCDEFGHIJKLM"
+ result = split_metadata_string(text, 3)
+ print(result)
+ >>> ['ABC', 'DEF', 'GHI', 'JKL']
+ ```
+
+ Args:
+ text (str): Text that will be split into chunks.
+ chunk_length (int): Single chunk size. Default chunk_length is
+ set to global variable `TVPAINT_CHUNK_LENGTH`.
+
+ Returns:
+ list: List of strings with at least one item.
+ """
+ if chunk_length is None:
+ chunk_length = TVPAINT_CHUNK_LENGTH
+ chunks = []
+ for idx in range(chunk_length, len(text) + chunk_length, chunk_length):
+ start_idx = idx - chunk_length
+ chunks.append(text[start_idx:idx])
+ return chunks
+
+
+def get_workfile_metadata_string_for_keys(metadata_keys):
+ """Read metadata for specific keys from current project workfile.
+
+ All values from entered keys are stored to single string without separator.
+
+ Function is designed to help get all values for one metadata key at once.
+ So order of passed keys matteres.
+
+ Args:
+ metadata_keys (list, str): Metadata keys for which data should be
+ retrieved. Order of keys matters! It is possible to enter only
+ single key as string.
+ """
+ # Add ability to pass only single key
+ if isinstance(metadata_keys, str):
+ metadata_keys = [metadata_keys]
+
+ output_file = tempfile.NamedTemporaryFile(
+ mode="w", prefix="a_tvp_", suffix=".txt", delete=False
+ )
+ output_file.close()
+ output_filepath = output_file.name.replace("\\", "/")
+
+ george_script_parts = []
+ george_script_parts.append(
+ "output_path = \"{}\"".format(output_filepath)
+ )
+ # Store data for each index of metadata key
+ for metadata_key in metadata_keys:
+ george_script_parts.append(
+ "tv_readprojectstring \"{}\" \"{}\" \"\"".format(
+ METADATA_SECTION, metadata_key
+ )
+ )
+ george_script_parts.append(
+ "tv_writetextfile \"strict\" \"append\" '\"'output_path'\"' result"
+ )
+
+ # Execute the script
+ george_script = "\n".join(george_script_parts)
+ execute_george_through_file(george_script)
+
+ # Load data from temp file
+ with open(output_filepath, "r") as stream:
+ file_content = stream.read()
+
+ # Remove `\n` from content
+ output_string = file_content.replace("\n", "")
+
+ # Delete temp file
+ os.remove(output_filepath)
+
+ return output_string
+
+
+def get_workfile_metadata_string(metadata_key):
+ """Read metadata for specific key from current project workfile."""
+ result = get_workfile_metadata_string_for_keys([metadata_key])
+ if not result:
+ return None
+
+ stripped_result = result.strip()
+ if not stripped_result:
+ return None
+
+ # NOTE Backwards compatibility when metadata key did not store range of key
+ # indexes but the value itself
+ # NOTE We don't have to care about negative values with `isdecimal` check
+ if not stripped_result.isdecimal():
+ metadata_string = result
+ else:
+ keys = []
+ for idx in range(int(stripped_result)):
+ keys.append("{}{}".format(metadata_key, idx))
+ metadata_string = get_workfile_metadata_string_for_keys(keys)
+
+ # Replace quotes plaholders with their values
+ metadata_string = (
+ metadata_string
+ .replace("{__sq__}", "'")
+ .replace("{__dq__}", "\"")
+ )
+ return metadata_string
+
+
+def get_workfile_metadata(metadata_key, default=None):
+ """Read and parse metadata for specific key from current project workfile.
+
+ Pipeline use function to store loaded and created instances within keys
+ stored in `SECTION_NAME_INSTANCES` and `SECTION_NAME_CONTAINERS`
+ constants.
+
+ Args:
+ metadata_key (str): Key defying which key should read. It is expected
+ value contain json serializable string.
+ """
+ if default is None:
+ default = []
+
+ json_string = get_workfile_metadata_string(metadata_key)
+ if json_string:
+ try:
+ return json.loads(json_string)
+ except json.decoder.JSONDecodeError:
+ # TODO remove when backwards compatibility of storing metadata
+ # will be removed
+ print((
+ "Fixed invalid metadata in workfile."
+ " Not serializable string was: {}"
+ ).format(json_string))
+ write_workfile_metadata(metadata_key, default)
+ return default
+
+
+def write_workfile_metadata(metadata_key, value):
+ """Write metadata for specific key into current project workfile.
+
+ George script has specific way how to work with quotes which should be
+ solved automatically with this function.
+
+ Args:
+ metadata_key (str): Key defying under which key value will be stored.
+ value (dict,list,str): Data to store they must be json serializable.
+ """
+ if isinstance(value, (dict, list)):
+ value = json.dumps(value)
+
+ if not value:
+ value = ""
+
+ # Handle quotes in dumped json string
+ # - replace single and double quotes with placeholders
+ value = (
+ value
+ .replace("'", "{__sq__}")
+ .replace("\"", "{__dq__}")
+ )
+ chunks = split_metadata_string(value)
+ chunks_len = len(chunks)
+
+ write_template = "tv_writeprojectstring \"{}\" \"{}\" \"{}\""
+ george_script_parts = []
+ # Add information about chunks length to metadata key itself
+ george_script_parts.append(
+ write_template.format(METADATA_SECTION, metadata_key, chunks_len)
+ )
+ # Add chunk values to indexed metadata keys
+ for idx, chunk_value in enumerate(chunks):
+ sub_key = "{}{}".format(metadata_key, idx)
+ george_script_parts.append(
+ write_template.format(METADATA_SECTION, sub_key, chunk_value)
+ )
+
+ george_script = "\n".join(george_script_parts)
+
+ return execute_george_through_file(george_script)
+
+
+def get_current_workfile_context():
+ """Return context in which was workfile saved."""
+ return get_workfile_metadata(SECTION_NAME_CONTEXT, {})
+
+
+def save_current_workfile_context(context):
+ """Save context which was used to create a workfile."""
+ return write_workfile_metadata(SECTION_NAME_CONTEXT, context)
+
+
+def remove_instance(instance):
+ """Remove instance from current workfile metadata."""
+ current_instances = get_workfile_metadata(SECTION_NAME_INSTANCES)
+ instance_id = instance.get("uuid")
+ found_idx = None
+ if instance_id:
+ for idx, _inst in enumerate(current_instances):
+ if _inst["uuid"] == instance_id:
+ found_idx = idx
+ break
+
+ if found_idx is None:
+ return
+ current_instances.pop(found_idx)
+ write_instances(current_instances)
+
+
+def list_instances():
+ """List all created instances from current workfile."""
+ return get_workfile_metadata(SECTION_NAME_INSTANCES)
+
+
+def write_instances(data):
+ return write_workfile_metadata(SECTION_NAME_INSTANCES, data)
+
+
+# Backwards compatibility
+def _write_instances(*args, **kwargs):
+ return write_instances(*args, **kwargs)
+
+
+def ls():
+ output = get_workfile_metadata(SECTION_NAME_CONTAINERS)
+ if output:
+ for item in output:
+ if "objectName" not in item and "members" in item:
+ members = item["members"]
+ if isinstance(members, list):
+ members = "|".join([str(member) for member in members])
+ item["objectName"] = members
+ return output
+
+
+def on_instance_toggle(instance, old_value, new_value):
+ """Update instance data in workfile on publish toggle."""
+ # Review may not have real instance in wokrfile metadata
+ if not instance.data.get("uuid"):
+ return
+
+ instance_id = instance.data["uuid"]
+ found_idx = None
+ current_instances = list_instances()
+ for idx, workfile_instance in enumerate(current_instances):
+ if workfile_instance["uuid"] == instance_id:
+ found_idx = idx
+ break
+
+ if found_idx is None:
+ return
+
+ if "active" in current_instances[found_idx]:
+ current_instances[found_idx]["active"] = new_value
+ write_instances(current_instances)
+
+
+def initial_launch():
+ # Setup project settings if its the template that's launched.
+ # TODO also check for template creation when it's possible to define
+ # templates
+ last_workfile = os.environ.get("AVALON_LAST_WORKFILE")
+ if not last_workfile or os.path.exists(last_workfile):
+ return
+
+ log.info("Setting up project...")
+ set_context_settings()
+
+
+def application_exit():
+ data = get_current_project_settings()
+ stop_timer = data["tvpaint"]["stop_timer_on_application_exit"]
+
+ if not stop_timer:
+ return
+
+ # Stop application timer.
+ webserver_url = os.environ.get("OPENPYPE_WEBSERVER_URL")
+ rest_api_url = "{}/timers_manager/stop_timer".format(webserver_url)
+ requests.post(rest_api_url)
+
+
+def set_context_settings(asset_doc=None):
+ """Set workfile settings by asset document data.
+
+ Change fps, resolution and frame start/end.
+ """
+
+ project_name = legacy_io.active_project()
+ if asset_doc is None:
+ asset_name = legacy_io.Session["AVALON_ASSET"]
+ # Use current session asset if not passed
+ asset_doc = get_asset_by_name(project_name, asset_name)
+
+ project_doc = get_project(project_name)
+
+ framerate = asset_doc["data"].get("fps")
+ if framerate is None:
+ framerate = project_doc["data"].get("fps")
+
+ if framerate is not None:
+ execute_george(
+ "tv_framerate {} \"timestretch\"".format(framerate)
+ )
+ else:
+ print("Framerate was not found!")
+
+ width_key = "resolutionWidth"
+ height_key = "resolutionHeight"
+
+ width = asset_doc["data"].get(width_key)
+ height = asset_doc["data"].get(height_key)
+ if width is None or height is None:
+ width = project_doc["data"].get(width_key)
+ height = project_doc["data"].get(height_key)
+
+ if width is None or height is None:
+ print("Resolution was not found!")
+ else:
+ execute_george(
+ "tv_resizepage {} {} 0".format(width, height)
+ )
+
+ frame_start = asset_doc["data"].get("frameStart")
+ frame_end = asset_doc["data"].get("frameEnd")
+
+ if frame_start is None or frame_end is None:
+ print("Frame range was not found!")
+ return
+
+ handles = asset_doc["data"].get("handles") or 0
+ handle_start = asset_doc["data"].get("handleStart")
+ handle_end = asset_doc["data"].get("handleEnd")
+
+ if handle_start is None or handle_end is None:
+ handle_start = handles
+ handle_end = handles
+
+ # Always start from 0 Mark In and set only Mark Out
+ mark_in = 0
+ mark_out = mark_in + (frame_end - frame_start) + handle_start + handle_end
+
+ execute_george("tv_markin {} set".format(mark_in))
+ execute_george("tv_markout {} set".format(mark_out))
diff --git a/openpype/hosts/tvpaint/api/plugin.py b/openpype/hosts/tvpaint/api/plugin.py
index e148e44a270..15ad8905e0e 100644
--- a/openpype/hosts/tvpaint/api/plugin.py
+++ b/openpype/hosts/tvpaint/api/plugin.py
@@ -1,8 +1,22 @@
-from openpype.api import PypeCreatorMixin
-from avalon.tvpaint import pipeline
+import re
+import uuid
+from openpype.pipeline import (
+ LegacyCreator,
+ LoaderPlugin,
+)
+from openpype.hosts.tvpaint.api import (
+ pipeline,
+ lib
+)
+
+
+class Creator(LegacyCreator):
+ def __init__(self, *args, **kwargs):
+ super(Creator, self).__init__(*args, **kwargs)
+ # Add unified identifier created with `uuid` module
+ self.data["uuid"] = str(uuid.uuid4())
-class Creator(PypeCreatorMixin, pipeline.Creator):
@classmethod
def get_dynamic_data(cls, *args, **kwargs):
dynamic_data = super(Creator, cls).get_dynamic_data(*args, **kwargs)
@@ -17,3 +31,95 @@ def get_dynamic_data(cls, *args, **kwargs):
if "task" not in dynamic_data and task_name:
dynamic_data["task"] = task_name
return dynamic_data
+
+ @staticmethod
+ def are_instances_same(instance_1, instance_2):
+ """Compare instances but skip keys with unique values.
+
+ During compare are skipped keys that will be 100% sure
+ different on new instance, like "id".
+
+ Returns:
+ bool: True if instances are same.
+ """
+ if (
+ not isinstance(instance_1, dict)
+ or not isinstance(instance_2, dict)
+ ):
+ return instance_1 == instance_2
+
+ checked_keys = set()
+ checked_keys.add("id")
+ for key, value in instance_1.items():
+ if key not in checked_keys:
+ if key not in instance_2:
+ return False
+ if value != instance_2[key]:
+ return False
+ checked_keys.add(key)
+
+ for key in instance_2.keys():
+ if key not in checked_keys:
+ return False
+ return True
+
+ def write_instances(self, data):
+ self.log.debug(
+ "Storing instance data to workfile. {}".format(str(data))
+ )
+ return pipeline.write_instances(data)
+
+ def process(self):
+ data = pipeline.list_instances()
+ data.append(self.data)
+ self.write_instances(data)
+
+
+class Loader(LoaderPlugin):
+ hosts = ["tvpaint"]
+
+ @staticmethod
+ def get_members_from_container(container):
+ if "members" not in container and "objectName" in container:
+ # Backwards compatibility
+ layer_ids_str = container.get("objectName")
+ return [
+ int(layer_id) for layer_id in layer_ids_str.split("|")
+ ]
+ return container["members"]
+
+ def get_unique_layer_name(self, asset_name, name):
+ """Layer name with counter as suffix.
+
+ Find higher 3 digit suffix from all layer names in scene matching regex
+ `{asset_name}_{name}_{suffix}`. Higher 3 digit suffix is used
+ as base for next number if scene does not contain layer matching regex
+ `0` is used ase base.
+
+ Args:
+ asset_name (str): Name of subset's parent asset document.
+ name (str): Name of loaded subset.
+
+ Returns:
+ (str): `{asset_name}_{name}_{higher suffix + 1}`
+ """
+ layer_name_base = "{}_{}".format(asset_name, name)
+
+ counter_regex = re.compile(r"_(\d{3})$")
+
+ higher_counter = 0
+ for layer in lib.get_layers_data():
+ layer_name = layer["name"]
+ if not layer_name.startswith(layer_name_base):
+ continue
+ number_subpart = layer_name[len(layer_name_base):]
+ groups = counter_regex.findall(number_subpart)
+ if len(groups) != 1:
+ continue
+
+ counter = int(groups[0])
+ if counter > higher_counter:
+ higher_counter = counter
+ continue
+
+ return "{}_{:0>3d}".format(layer_name_base, higher_counter + 1)
diff --git a/openpype/hosts/tvpaint/api/workio.py b/openpype/hosts/tvpaint/api/workio.py
new file mode 100644
index 00000000000..1a5ad00ca8b
--- /dev/null
+++ b/openpype/hosts/tvpaint/api/workio.py
@@ -0,0 +1,58 @@
+"""Host API required for Work Files.
+# TODO @iLLiCiT implement functions:
+ has_unsaved_changes
+"""
+
+from openpype.pipeline import (
+ HOST_WORKFILE_EXTENSIONS,
+ legacy_io,
+)
+from .lib import (
+ execute_george,
+ execute_george_through_file
+)
+from .pipeline import save_current_workfile_context
+
+
+def open_file(filepath):
+ """Open the scene file in Blender."""
+ george_script = "tv_LoadProject '\"'\"{}\"'\"'".format(
+ filepath.replace("\\", "/")
+ )
+ return execute_george_through_file(george_script)
+
+
+def save_file(filepath):
+ """Save the open scene file."""
+ # Store context to workfile before save
+ context = {
+ "project": legacy_io.Session["AVALON_PROJECT"],
+ "asset": legacy_io.Session["AVALON_ASSET"],
+ "task": legacy_io.Session["AVALON_TASK"]
+ }
+ save_current_workfile_context(context)
+
+ # Execute george script to save workfile.
+ george_script = "tv_SaveProject {}".format(filepath.replace("\\", "/"))
+ return execute_george(george_script)
+
+
+def current_file():
+ """Return the path of the open scene file."""
+ george_script = "tv_GetProjectName"
+ return execute_george(george_script)
+
+
+def has_unsaved_changes():
+ """Does the open scene file have unsaved changes?"""
+ return False
+
+
+def file_extensions():
+ """Return the supported file extensions for Blender scene files."""
+ return HOST_WORKFILE_EXTENSIONS["tvpaint"]
+
+
+def work_root(session):
+ """Return the default root to browse for work files."""
+ return session["AVALON_WORKDIR"]
diff --git a/openpype/hosts/tvpaint/hooks/pre_launch_args.py b/openpype/hosts/tvpaint/hooks/pre_launch_args.py
index b0b13529ca7..c31403437ab 100644
--- a/openpype/hosts/tvpaint/hooks/pre_launch_args.py
+++ b/openpype/hosts/tvpaint/hooks/pre_launch_args.py
@@ -1,14 +1,8 @@
-import os
-import shutil
-
-from openpype.hosts import tvpaint
from openpype.lib import (
PreLaunchHook,
- get_pype_execute_args
+ get_openpype_execute_args
)
-import avalon
-
class TvpaintPrelaunchHook(PreLaunchHook):
"""Launch arguments preparation.
@@ -30,7 +24,7 @@ def execute(self):
while self.launch_context.launch_args:
remainders.append(self.launch_context.launch_args.pop(0))
- new_launch_args = get_pype_execute_args(
+ new_launch_args = get_openpype_execute_args(
"run", self.launch_script_path(), executable_path
)
@@ -44,10 +38,6 @@ def execute(self):
self.launch_context.launch_args.extend(remainders)
def launch_script_path(self):
- avalon_dir = os.path.dirname(os.path.abspath(avalon.__file__))
- script_path = os.path.join(
- avalon_dir,
- "tvpaint",
- "launch_script.py"
- )
- return script_path
\ No newline at end of file
+ from openpype.hosts.tvpaint import get_launch_script_path
+
+ return get_launch_script_path()
diff --git a/openpype/hosts/tvpaint/lib.py b/openpype/hosts/tvpaint/lib.py
index 513bb2d952e..c67ab1e4fb6 100644
--- a/openpype/hosts/tvpaint/lib.py
+++ b/openpype/hosts/tvpaint/lib.py
@@ -278,7 +278,7 @@ def _cleanup_out_range_frames(output_idx_by_frame_idx, range_start, range_end):
}
// Result
{
- 2: 2, // Redirect to self as is first that refence out range
+ 2: 2, // Redirect to self as is first that reference out range
3: 2 // Redirect to first redirected frame
}
```
@@ -573,7 +573,7 @@ def composite_rendered_layers(
layer_ids_by_position[layer_position] = layer["layer_id"]
# Sort layer positions
- sorted_positions = tuple(sorted(layer_ids_by_position.keys()))
+ sorted_positions = tuple(reversed(sorted(layer_ids_by_position.keys())))
# Prepare variable where filepaths without any rendered content
# - transparent will be created
transparent_filepaths = set()
@@ -593,7 +593,7 @@ def composite_rendered_layers(
transparent_filepaths.add(dst_filepath)
continue
- # Store first destionation filepath to be used for transparent images
+ # Store first destination filepath to be used for transparent images
if first_dst_filepath is None:
first_dst_filepath = dst_filepath
@@ -657,7 +657,7 @@ def rename_filepaths_by_frame_start(
max(range_end, new_frame_end)
)
- # Use differnet ranges based on Mark In and output Frame Start values
+ # Use different ranges based on Mark In and output Frame Start values
# - this is to make sure that filename renaming won't affect files that
# are not renamed yet
if range_start < new_frame_start:
diff --git a/openpype/hosts/tvpaint/plugins/create/create_render_layer.py b/openpype/hosts/tvpaint/plugins/create/create_render_layer.py
index af6c0f0eee6..3b5bd471897 100644
--- a/openpype/hosts/tvpaint/plugins/create/create_render_layer.py
+++ b/openpype/hosts/tvpaint/plugins/create/create_render_layer.py
@@ -1,11 +1,11 @@
-from avalon.api import CreatorError
-from avalon.tvpaint import (
+from openpype.pipeline import CreatorError
+from openpype.lib import prepare_template_data
+from openpype.hosts.tvpaint.api import (
+ plugin,
pipeline,
lib,
CommunicationWrapper
)
-from openpype.hosts.tvpaint.api import plugin
-from openpype.lib import prepare_template_data
class CreateRenderlayer(plugin.Creator):
@@ -24,7 +24,9 @@ class CreateRenderlayer(plugin.Creator):
" {clip_id} {group_id} {r} {g} {b} \"{name}\""
)
- dynamic_subset_keys = ["render_pass", "render_layer", "group"]
+ dynamic_subset_keys = [
+ "renderpass", "renderlayer", "render_pass", "render_layer", "group"
+ ]
@classmethod
def get_dynamic_data(
@@ -34,12 +36,17 @@ def get_dynamic_data(
variant, task_name, asset_id, project_name, host_name
)
# Use render pass name from creator's plugin
- dynamic_data["render_pass"] = cls.render_pass
+ dynamic_data["renderpass"] = cls.render_pass
# Add variant to render layer
- dynamic_data["render_layer"] = variant
+ dynamic_data["renderlayer"] = variant
# Change family for subset name fill
dynamic_data["family"] = "render"
+ # TODO remove - Backwards compatibility for old subset name templates
+ # - added 2022/04/28
+ dynamic_data["render_pass"] = dynamic_data["renderpass"]
+ dynamic_data["render_layer"] = dynamic_data["renderlayer"]
+
return dynamic_data
@classmethod
@@ -56,7 +63,7 @@ def get_default_variant(cls):
# Validate that communication is initialized
if CommunicationWrapper.communicator:
# Get currently selected layers
- layers_data = lib.layers_data()
+ layers_data = lib.get_layers_data()
selected_layers = [
layer
@@ -75,7 +82,7 @@ def get_default_variant(cls):
def process(self):
self.log.debug("Query data from workfile.")
instances = pipeline.list_instances()
- layers_data = lib.layers_data()
+ layers_data = lib.get_layers_data()
self.log.debug("Checking for selection groups.")
# Collect group ids from selection
@@ -102,7 +109,7 @@ def process(self):
self.log.debug(f"Selected group id is \"{group_id}\".")
self.data["group_id"] = group_id
- group_data = lib.groups_data()
+ group_data = lib.get_groups_data()
group_name = None
for group in group_data:
if group["group_id"] == group_id:
@@ -169,7 +176,7 @@ def process(self):
return
self.log.debug("Querying groups data from workfile.")
- groups_data = lib.groups_data()
+ groups_data = lib.get_groups_data()
self.log.debug("Changing name of the group.")
selected_group = None
@@ -196,6 +203,7 @@ def process(self):
)
def _ask_user_subset_override(self, instance):
+ from Qt import QtCore
from Qt.QtWidgets import QMessageBox
title = "Subset \"{}\" already exist".format(instance["subset"])
@@ -205,6 +213,10 @@ def _ask_user_subset_override(self, instance):
).format(instance["subset"])
dialog = QMessageBox()
+ dialog.setWindowFlags(
+ dialog.windowFlags()
+ | QtCore.Qt.WindowStaysOnTopHint
+ )
dialog.setWindowTitle(title)
dialog.setText(text)
dialog.setStandardButtons(QMessageBox.Yes | QMessageBox.No)
diff --git a/openpype/hosts/tvpaint/plugins/create/create_render_pass.py b/openpype/hosts/tvpaint/plugins/create/create_render_pass.py
index ad06520210d..26fa8ac51aa 100644
--- a/openpype/hosts/tvpaint/plugins/create/create_render_pass.py
+++ b/openpype/hosts/tvpaint/plugins/create/create_render_pass.py
@@ -1,11 +1,11 @@
-from avalon.api import CreatorError
-from avalon.tvpaint import (
+from openpype.pipeline import CreatorError
+from openpype.lib import prepare_template_data
+from openpype.hosts.tvpaint.api import (
+ plugin,
pipeline,
lib,
CommunicationWrapper
)
-from openpype.hosts.tvpaint.api import plugin
-from openpype.lib import prepare_template_data
class CreateRenderPass(plugin.Creator):
@@ -20,7 +20,9 @@ class CreateRenderPass(plugin.Creator):
icon = "cube"
defaults = ["Main"]
- dynamic_subset_keys = ["render_pass", "render_layer"]
+ dynamic_subset_keys = [
+ "renderpass", "renderlayer", "render_pass", "render_layer"
+ ]
@classmethod
def get_dynamic_data(
@@ -29,9 +31,13 @@ def get_dynamic_data(
dynamic_data = super(CreateRenderPass, cls).get_dynamic_data(
variant, task_name, asset_id, project_name, host_name
)
- dynamic_data["render_pass"] = variant
+ dynamic_data["renderpass"] = variant
dynamic_data["family"] = "render"
+ # TODO remove - Backwards compatibility for old subset name templates
+ # - added 2022/04/28
+ dynamic_data["render_pass"] = dynamic_data["renderpass"]
+
return dynamic_data
@classmethod
@@ -115,6 +121,7 @@ def process(self):
else:
render_layer = beauty_instance["variant"]
+ subset_name_fill_data["renderlayer"] = render_layer
subset_name_fill_data["render_layer"] = render_layer
# Format dynamic keys in subset name
@@ -129,7 +136,7 @@ def process(self):
self.data["group_id"] = group_id
self.data["pass"] = variant
- self.data["render_layer"] = render_layer
+ self.data["renderlayer"] = render_layer
# Collect selected layer ids to be stored into instance
layer_names = [layer["name"] for layer in selected_layers]
diff --git a/openpype/hosts/tvpaint/plugins/load/load_image.py b/openpype/hosts/tvpaint/plugins/load/load_image.py
index f77fab87f8b..f861d0119e0 100644
--- a/openpype/hosts/tvpaint/plugins/load/load_image.py
+++ b/openpype/hosts/tvpaint/plugins/load/load_image.py
@@ -1,11 +1,11 @@
-from avalon.vendor import qargparse
-from avalon.tvpaint import lib, pipeline
+import qargparse
+from openpype.hosts.tvpaint.api import lib, plugin
-class ImportImage(pipeline.Loader):
+class ImportImage(plugin.Loader):
"""Load image or image sequence to TVPaint as new layer."""
- families = ["render", "image", "background", "plate"]
+ families = ["render", "image", "background", "plate", "review"]
representations = ["*"]
label = "Import Image"
diff --git a/openpype/hosts/tvpaint/plugins/load/load_reference_image.py b/openpype/hosts/tvpaint/plugins/load/load_reference_image.py
index b8b20ed20aa..af1a4a9b6b3 100644
--- a/openpype/hosts/tvpaint/plugins/load/load_reference_image.py
+++ b/openpype/hosts/tvpaint/plugins/load/load_reference_image.py
@@ -1,13 +1,13 @@
import collections
-from avalon.pipeline import get_representation_context
-from avalon.vendor import qargparse
-from avalon.tvpaint import lib, pipeline
+import qargparse
+from openpype.pipeline import get_representation_context
+from openpype.hosts.tvpaint.api import lib, pipeline, plugin
-class LoadImage(pipeline.Loader):
+class LoadImage(plugin.Loader):
"""Load image or image sequence to TVPaint as new layer."""
- families = ["render", "image", "background", "plate"]
+ families = ["render", "image", "background", "plate", "review"]
representations = ["*"]
label = "Load Image"
diff --git a/openpype/hosts/tvpaint/plugins/load/load_sound.py b/openpype/hosts/tvpaint/plugins/load/load_sound.py
index c83748fe06c..3f42370f5c8 100644
--- a/openpype/hosts/tvpaint/plugins/load/load_sound.py
+++ b/openpype/hosts/tvpaint/plugins/load/load_sound.py
@@ -1,9 +1,9 @@
import os
import tempfile
-from avalon.tvpaint import lib, pipeline
+from openpype.hosts.tvpaint.api import lib, plugin
-class ImportSound(pipeline.Loader):
+class ImportSound(plugin.Loader):
"""Load sound to TVPaint.
Sound layers does not have ids but only position index so we can't
diff --git a/openpype/hosts/tvpaint/plugins/load/load_workfile.py b/openpype/hosts/tvpaint/plugins/load/load_workfile.py
index f410a1ab9db..a99b3007301 100644
--- a/openpype/hosts/tvpaint/plugins/load/load_workfile.py
+++ b/openpype/hosts/tvpaint/plugins/load/load_workfile.py
@@ -1,16 +1,20 @@
-import getpass
import os
-from avalon.tvpaint import lib, pipeline, get_current_workfile_context
-from avalon import api, io
-from openpype.lib import (
+from openpype.lib import StringTemplate
+from openpype.pipeline import (
+ registered_host,
+ legacy_io,
+ Anatomy,
+)
+from openpype.pipeline.workfile import (
get_workfile_template_key_from_context,
- get_workdir_data
+ get_last_workfile_with_version,
)
-from openpype.api import Anatomy
+from openpype.pipeline.template_data import get_template_data_with_names
+from openpype.hosts.tvpaint.api import lib, pipeline, plugin
-class LoadWorkfile(pipeline.Loader):
+class LoadWorkfile(plugin.Loader):
"""Load workfile."""
families = ["workfile"]
@@ -21,10 +25,10 @@ class LoadWorkfile(pipeline.Loader):
def load(self, context, name, namespace, options):
# Load context of current workfile as first thing
# - which context and extension has
- host = api.registered_host()
+ host = registered_host()
current_file = host.current_file()
- context = get_current_workfile_context()
+ context = pipeline.get_current_workfile_context()
filepath = self.fname.replace("\\", "/")
@@ -40,36 +44,29 @@ def load(self, context, name, namespace, options):
# Save workfile.
host_name = "tvpaint"
+ project_name = context.get("project")
asset_name = context.get("asset")
task_name = context.get("task")
# Far cases when there is workfile without context
if not asset_name:
- asset_name = io.Session["AVALON_ASSET"]
- task_name = io.Session["AVALON_TASK"]
-
- project_doc = io.find_one({
- "type": "project"
- })
- asset_doc = io.find_one({
- "type": "asset",
- "name": asset_name
- })
- project_name = project_doc["name"]
+ project_name = legacy_io.active_project()
+ asset_name = legacy_io.Session["AVALON_ASSET"]
+ task_name = legacy_io.Session["AVALON_TASK"]
template_key = get_workfile_template_key_from_context(
asset_name,
task_name,
host_name,
- project_name=project_name,
- dbcon=io
+ project_name=project_name
)
anatomy = Anatomy(project_name)
- data = get_workdir_data(project_doc, asset_doc, task_name, host_name)
+ data = get_template_data_with_names(
+ project_name, asset_name, task_name, host_name
+ )
data["root"] = anatomy.roots
- data["user"] = getpass.getuser()
- template = anatomy.templates[template_key]["file"]
+ file_template = anatomy.templates[template_key]["file"]
# Define saving file extension
if current_file:
@@ -81,11 +78,12 @@ def load(self, context, name, namespace, options):
data["ext"] = extension
- work_root = api.format_template_with_optional_keys(
- data, anatomy.templates[template_key]["folder"]
+ folder_template = anatomy.templates[template_key]["folder"]
+ work_root = StringTemplate.format_strict_template(
+ folder_template, data
)
- version = api.last_workfile_with_version(
- work_root, template, data, host.file_extensions()
+ version = get_last_workfile_with_version(
+ work_root, file_template, data, host.file_extensions()
)[1]
if version is None:
@@ -95,8 +93,8 @@ def load(self, context, name, namespace, options):
data["version"] = version
- path = os.path.join(
- work_root,
- api.format_template_with_optional_keys(data, template)
+ filename = StringTemplate.format_strict_template(
+ file_template, data
)
+ path = os.path.join(work_root, filename)
host.save_file(path)
diff --git a/openpype/hosts/tvpaint/plugins/publish/collect_instances.py b/openpype/hosts/tvpaint/plugins/publish/collect_instances.py
index 1d7a48e389e..ae1326a5bde 100644
--- a/openpype/hosts/tvpaint/plugins/publish/collect_instances.py
+++ b/openpype/hosts/tvpaint/plugins/publish/collect_instances.py
@@ -1,10 +1,10 @@
-import os
import json
import copy
import pyblish.api
-from avalon import io
-from openpype.lib import get_subset_name_with_asset_doc
+from openpype.client import get_asset_by_name
+from openpype.pipeline import legacy_io
+from openpype.pipeline.create import get_subset_name
class CollectInstances(pyblish.api.ContextPlugin):
@@ -20,30 +20,57 @@ def process(self, context):
json.dumps(workfile_instances, indent=4)
))
+ filtered_instance_data = []
# Backwards compatibility for workfiles that already have review
# instance in metadata.
review_instance_exist = False
for instance_data in workfile_instances:
- if instance_data["family"] == "review":
+ family = instance_data["family"]
+ if family == "review":
review_instance_exist = True
- break
+
+ elif family not in ("renderPass", "renderLayer"):
+ self.log.info("Unknown family \"{}\". Skipping {}".format(
+ family, json.dumps(instance_data, indent=4)
+ ))
+ continue
+
+ filtered_instance_data.append(instance_data)
# Fake review instance if review was not found in metadata families
if not review_instance_exist:
- workfile_instances.append(
+ filtered_instance_data.append(
self._create_review_instance_data(context)
)
- for instance_data in workfile_instances:
+ for instance_data in filtered_instance_data:
instance_data["fps"] = context.data["sceneFps"]
+ # Conversion from older instances
+ # - change 'render_layer' to 'renderlayer'
+ render_layer = instance_data.get("instance_data")
+ if not render_layer:
+ # Render Layer has only variant
+ if instance_data["family"] == "renderLayer":
+ render_layer = instance_data.get("variant")
+
+ # Backwards compatibility for renderPasses
+ elif "render_layer" in instance_data:
+ render_layer = instance_data["render_layer"]
+
+ if render_layer:
+ instance_data["renderlayer"] = render_layer
+
# Store workfile instance data to instance data
instance_data["originData"] = copy.deepcopy(instance_data)
# Global instance data modifications
# Fill families
family = instance_data["family"]
+ families = [family]
+ if family != "review":
+ families.append("review")
# Add `review` family for thumbnail integration
- instance_data["families"] = [family, "review"]
+ instance_data["families"] = families
# Instance name
subset_name = instance_data["subset"]
@@ -66,29 +93,28 @@ def process(self, context):
if family == "review":
# Change subset name of review instance
+ # Project name from workfile context
+ project_name = context.data["workfile_context"]["project"]
+
# Collect asset doc to get asset id
# - not sure if it's good idea to require asset id in
# get_subset_name?
asset_name = context.data["workfile_context"]["asset"]
- asset_doc = io.find_one({
- "type": "asset",
- "name": asset_name
- })
+ asset_doc = get_asset_by_name(project_name, asset_name)
- # Project name from workfile context
- project_name = context.data["workfile_context"]["project"]
- # Host name from environemnt variable
- host_name = os.environ["AVALON_APP"]
+ # Host name from environment variable
+ host_name = context.data["hostName"]
# Use empty variant value
variant = ""
- task_name = io.Session["AVALON_TASK"]
- new_subset_name = get_subset_name_with_asset_doc(
+ task_name = legacy_io.Session["AVALON_TASK"]
+ new_subset_name = get_subset_name(
family,
variant,
task_name,
asset_doc,
project_name,
- host_name
+ host_name,
+ project_settings=context.data["project_settings"]
)
instance_data["subset"] = new_subset_name
@@ -106,12 +132,6 @@ def process(self, context):
instance = self.create_render_pass_instance(
context, instance_data
)
- else:
- raise AssertionError(
- "Instance with unknown family \"{}\": {}".format(
- family, instance_data
- )
- )
if instance is None:
continue
@@ -151,7 +171,7 @@ def create_render_layer_instance(self, context, instance_data):
# Change subset name
# Final family of an instance will be `render`
new_family = "render"
- task_name = io.Session["AVALON_TASK"]
+ task_name = legacy_io.Session["AVALON_TASK"]
new_subset_name = "{}{}_{}_Beauty".format(
new_family, task_name.capitalize(), name
)
@@ -186,7 +206,7 @@ def create_render_pass_instance(self, context, instance_data):
"Creating render pass instance. \"{}\"".format(pass_name)
)
# Change label
- render_layer = instance_data["render_layer"]
+ render_layer = instance_data["renderlayer"]
# Backwards compatibility
# - subset names were not stored as final subset names during creation
@@ -196,7 +216,7 @@ def create_render_pass_instance(self, context, instance_data):
# Final family of an instance will be `render`
new_family = "render"
old_subset_name = instance_data["subset"]
- task_name = io.Session["AVALON_TASK"]
+ task_name = legacy_io.Session["AVALON_TASK"]
new_subset_name = "{}{}_{}_{}".format(
new_family, task_name.capitalize(), render_layer, pass_name
)
@@ -218,7 +238,7 @@ def create_render_pass_instance(self, context, instance_data):
# - not 100% working as it was found out that layer ids can't be
# used as unified identifier across multiple workstations
layers_by_id = {
- layer["id"]: layer
+ layer["layer_id"]: layer
for layer in layers_data
}
layer_ids = instance_data["layer_ids"]
diff --git a/openpype/hosts/tvpaint/plugins/publish/collect_scene_render.py b/openpype/hosts/tvpaint/plugins/publish/collect_scene_render.py
new file mode 100644
index 00000000000..92a2815ba06
--- /dev/null
+++ b/openpype/hosts/tvpaint/plugins/publish/collect_scene_render.py
@@ -0,0 +1,114 @@
+import json
+import copy
+import pyblish.api
+
+from openpype.client import get_asset_by_name
+from openpype.pipeline.create import get_subset_name
+
+
+class CollectRenderScene(pyblish.api.ContextPlugin):
+ """Collect instance which renders whole scene in PNG.
+
+ Creates instance with family 'renderScene' which will have all layers
+ to render which will be composite into one result. The instance is not
+ collected from scene.
+
+ Scene will be rendered with all visible layers similar way like review is.
+
+ Instance is disabled if there are any created instances of 'renderLayer'
+ or 'renderPass'. That is because it is expected that this instance is
+ used as lazy publish of TVPaint file.
+
+ Subset name is created similar way like 'renderLayer' family. It can use
+ `renderPass` and `renderLayer` keys which can be set using settings and
+ `variant` is filled using `renderPass` value.
+ """
+ label = "Collect Render Scene"
+ order = pyblish.api.CollectorOrder - 0.39
+ hosts = ["tvpaint"]
+
+ # Value of 'render_pass' in subset name template
+ render_pass = "beauty"
+
+ # Settings attributes
+ enabled = False
+ # Value of 'render_layer' and 'variant' in subset name template
+ render_layer = "Main"
+
+ def process(self, context):
+ # Check if there are created instances of renderPass and renderLayer
+ # - that will define if renderScene instance is enabled after
+ # collection
+ any_created_instance = False
+ for instance in context:
+ family = instance.data["family"]
+ if family in ("renderPass", "renderLayer"):
+ any_created_instance = True
+ break
+
+ # Global instance data modifications
+ # Fill families
+ family = "renderScene"
+ # Add `review` family for thumbnail integration
+ families = [family, "review"]
+
+ # Collect asset doc to get asset id
+ # - not sure if it's good idea to require asset id in
+ # get_subset_name?
+ workfile_context = context.data["workfile_context"]
+ # Project name from workfile context
+ project_name = context.data["workfile_context"]["project"]
+ asset_name = workfile_context["asset"]
+ asset_doc = get_asset_by_name(project_name, asset_name)
+
+ # Host name from environment variable
+ host_name = context.data["hostName"]
+ # Variant is using render pass name
+ variant = self.render_layer
+ dynamic_data = {
+ "renderlayer": self.render_layer,
+ "renderpass": self.render_pass,
+ }
+ # TODO remove - Backwards compatibility for old subset name templates
+ # - added 2022/04/28
+ dynamic_data["render_layer"] = dynamic_data["renderlayer"]
+ dynamic_data["render_pass"] = dynamic_data["renderpass"]
+
+ task_name = workfile_context["task"]
+ subset_name = get_subset_name(
+ "render",
+ variant,
+ task_name,
+ asset_doc,
+ project_name,
+ host_name,
+ dynamic_data=dynamic_data,
+ project_settings=context.data["project_settings"]
+ )
+
+ instance_data = {
+ "family": family,
+ "families": families,
+ "fps": context.data["sceneFps"],
+ "subset": subset_name,
+ "name": subset_name,
+ "label": "{} [{}-{}]".format(
+ subset_name,
+ context.data["sceneMarkIn"] + 1,
+ context.data["sceneMarkOut"] + 1
+ ),
+ "active": not any_created_instance,
+ "publish": not any_created_instance,
+ "representations": [],
+ "layers": copy.deepcopy(context.data["layersData"]),
+ "asset": asset_name,
+ "task": task_name,
+ # Add render layer to instance data
+ "renderlayer": self.render_layer
+ }
+
+ instance = context.create_instance(**instance_data)
+
+ self.log.debug("Created instance: {}\n{}".format(
+ instance, json.dumps(instance.data, indent=4)
+ ))
diff --git a/openpype/hosts/tvpaint/plugins/publish/collect_workfile.py b/openpype/hosts/tvpaint/plugins/publish/collect_workfile.py
index 68ba350a85d..8c7c8c3899f 100644
--- a/openpype/hosts/tvpaint/plugins/publish/collect_workfile.py
+++ b/openpype/hosts/tvpaint/plugins/publish/collect_workfile.py
@@ -1,9 +1,10 @@
import os
import json
import pyblish.api
-from avalon import io
-from openpype.lib import get_subset_name_with_asset_doc
+from openpype.client import get_asset_by_name
+from openpype.pipeline import legacy_io
+from openpype.pipeline.create import get_subset_name
class CollectWorkfile(pyblish.api.ContextPlugin):
@@ -22,31 +23,30 @@ def process(self, context):
basename, ext = os.path.splitext(filename)
instance = context.create_instance(name=basename)
+ # Project name from workfile context
+ project_name = context.data["workfile_context"]["project"]
+
# Get subset name of workfile instance
# Collect asset doc to get asset id
# - not sure if it's good idea to require asset id in
# get_subset_name?
family = "workfile"
asset_name = context.data["workfile_context"]["asset"]
- asset_doc = io.find_one({
- "type": "asset",
- "name": asset_name
- })
+ asset_doc = get_asset_by_name(project_name, asset_name)
- # Project name from workfile context
- project_name = context.data["workfile_context"]["project"]
- # Host name from environemnt variable
+ # Host name from environment variable
host_name = os.environ["AVALON_APP"]
# Use empty variant value
variant = ""
- task_name = io.Session["AVALON_TASK"]
- subset_name = get_subset_name_with_asset_doc(
+ task_name = legacy_io.Session["AVALON_TASK"]
+ subset_name = get_subset_name(
family,
variant,
task_name,
asset_doc,
project_name,
- host_name
+ host_name,
+ project_settings=context.data["project_settings"]
)
# Create Workfile instance
diff --git a/openpype/hosts/tvpaint/plugins/publish/collect_workfile_data.py b/openpype/hosts/tvpaint/plugins/publish/collect_workfile_data.py
index f4259f1b5f4..c59ef82f853 100644
--- a/openpype/hosts/tvpaint/plugins/publish/collect_workfile_data.py
+++ b/openpype/hosts/tvpaint/plugins/publish/collect_workfile_data.py
@@ -3,8 +3,9 @@
import tempfile
import pyblish.api
-import avalon.api
-from avalon.tvpaint import pipeline, lib
+
+from openpype.pipeline import legacy_io
+from openpype.hosts.tvpaint.api import pipeline, lib
class ResetTVPaintWorkfileMetadata(pyblish.api.Action):
@@ -49,9 +50,9 @@ def process(self, context):
# Collect and store current context to have reference
current_context = {
- "project": avalon.api.Session["AVALON_PROJECT"],
- "asset": avalon.api.Session["AVALON_ASSET"],
- "task": avalon.api.Session["AVALON_TASK"]
+ "project": legacy_io.Session["AVALON_PROJECT"],
+ "asset": legacy_io.Session["AVALON_ASSET"],
+ "task": legacy_io.Session["AVALON_TASK"]
}
context.data["previous_context"] = current_context
self.log.debug("Current context is: {}".format(current_context))
@@ -69,7 +70,7 @@ def process(self, context):
("AVALON_TASK", "task")
)
for env_key, key in key_map:
- avalon.api.Session[env_key] = workfile_context[key]
+ legacy_io.Session[env_key] = workfile_context[key]
os.environ[env_key] = workfile_context[key]
self.log.info("Context changed to: {}".format(workfile_context))
diff --git a/openpype/hosts/tvpaint/plugins/publish/extract_convert_to_exr.py b/openpype/hosts/tvpaint/plugins/publish/extract_convert_to_exr.py
new file mode 100644
index 00000000000..ab5bbc5e2c7
--- /dev/null
+++ b/openpype/hosts/tvpaint/plugins/publish/extract_convert_to_exr.py
@@ -0,0 +1,99 @@
+"""Plugin converting png files from ExtractSequence into exrs.
+
+Requires:
+ ExtractSequence - source of PNG
+ ExtractReview - review was already created so we can convert to any exr
+"""
+import os
+import json
+
+import pyblish.api
+from openpype.lib import (
+ get_oiio_tools_path,
+ run_subprocess,
+)
+from openpype.pipeline import KnownPublishError
+
+
+class ExtractConvertToEXR(pyblish.api.InstancePlugin):
+ # Offset to get after ExtractSequence plugin.
+ order = pyblish.api.ExtractorOrder + 0.1
+ label = "Extract Sequence EXR"
+ hosts = ["tvpaint"]
+ families = ["render"]
+
+ enabled = False
+
+ # Replace source PNG files or just add
+ replace_pngs = True
+ # EXR compression
+ exr_compression = "ZIP"
+
+ def process(self, instance):
+ repres = instance.data.get("representations")
+ if not repres:
+ return
+
+ oiio_path = get_oiio_tools_path()
+ # Raise an exception when oiiotool is not available
+ # - this can currently happen on MacOS machines
+ if not os.path.exists(oiio_path):
+ KnownPublishError(
+ "OpenImageIO tool is not available on this machine."
+ )
+
+ new_repres = []
+ for repre in repres:
+ if repre["name"] != "png":
+ continue
+
+ self.log.info(
+ "Processing representation: {}".format(
+ json.dumps(repre, sort_keys=True, indent=4)
+ )
+ )
+
+ src_filepaths = set()
+ new_filenames = []
+ for src_filename in repre["files"]:
+ dst_filename = os.path.splitext(src_filename)[0] + ".exr"
+ new_filenames.append(dst_filename)
+
+ src_filepath = os.path.join(repre["stagingDir"], src_filename)
+ dst_filepath = os.path.join(repre["stagingDir"], dst_filename)
+
+ src_filepaths.add(src_filepath)
+
+ args = [
+ oiio_path, src_filepath,
+ "--compression", self.exr_compression,
+ # TODO how to define color conversion?
+ "--colorconvert", "sRGB", "linear",
+ "-o", dst_filepath
+ ]
+ run_subprocess(args)
+
+ new_repres.append(
+ {
+ "name": "exr",
+ "ext": "exr",
+ "files": new_filenames,
+ "stagingDir": repre["stagingDir"],
+ "tags": list(repre["tags"])
+ }
+ )
+
+ if self.replace_pngs:
+ instance.data["representations"].remove(repre)
+
+ for filepath in src_filepaths:
+ instance.context.data["cleanupFullPaths"].append(filepath)
+
+ instance.data["representations"].extend(new_repres)
+ self.log.info(
+ "Representations: {}".format(
+ json.dumps(
+ instance.data["representations"], sort_keys=True, indent=4
+ )
+ )
+ )
diff --git a/openpype/hosts/tvpaint/plugins/publish/extract_sequence.py b/openpype/hosts/tvpaint/plugins/publish/extract_sequence.py
index 6235b6211d9..77712347bd8 100644
--- a/openpype/hosts/tvpaint/plugins/publish/extract_sequence.py
+++ b/openpype/hosts/tvpaint/plugins/publish/extract_sequence.py
@@ -2,23 +2,23 @@
import copy
import tempfile
+from PIL import Image
+
import pyblish.api
-from avalon.tvpaint import lib
-from openpype.hosts.tvpaint.api.lib import composite_images
+from openpype.hosts.tvpaint.api import lib
from openpype.hosts.tvpaint.lib import (
calculate_layers_extraction_data,
get_frame_filename_template,
fill_reference_frames,
composite_rendered_layers,
- rename_filepaths_by_frame_start
+ rename_filepaths_by_frame_start,
)
-from PIL import Image
class ExtractSequence(pyblish.api.Extractor):
label = "Extract Sequence"
hosts = ["tvpaint"]
- families = ["review", "renderPass", "renderLayer"]
+ families = ["review", "renderPass", "renderLayer", "renderScene"]
# Modifiable with settings
review_bg = [255, 255, 255, 255]
@@ -73,14 +73,8 @@ def process(self, instance):
scene_bg_color = instance.context.data["sceneBgColor"]
- # --- Fallbacks ----------------------------------------------------
- # This is required if validations of ranges are ignored.
- # - all of this code won't change processing if range to render
- # match to range of expected output
-
# Prepare output frames
output_frame_start = frame_start - handle_start
- output_frame_end = frame_end + handle_end
# Change output frame start to 0 if handles cause it's negative number
if output_frame_start < 0:
@@ -90,32 +84,8 @@ def process(self, instance):
).format(frame_start, handle_start))
output_frame_start = 0
- # Check Marks range and output range
- output_range = output_frame_end - output_frame_start
- marks_range = mark_out - mark_in
-
- # Lower Mark Out if mark range is bigger than output
- # - do not rendered not used frames
- if output_range < marks_range:
- new_mark_out = mark_out - (marks_range - output_range)
- self.log.warning((
- "Lowering render range to {} frames. Changed Mark Out {} -> {}"
- ).format(marks_range + 1, mark_out, new_mark_out))
- # Assign new mark out to variable
- mark_out = new_mark_out
-
- # Lower output frame end so representation has right `frameEnd` value
- elif output_range > marks_range:
- new_output_frame_end = (
- output_frame_end - (output_range - marks_range)
- )
- self.log.warning((
- "Lowering representation range to {} frames."
- " Changed frame end {} -> {}"
- ).format(output_range + 1, mark_out, new_output_frame_end))
- output_frame_end = new_output_frame_end
-
- # -------------------------------------------------------------------
+ # Calculate frame end
+ output_frame_end = output_frame_start + (mark_out - mark_in)
# Save to staging dir
output_dir = instance.data.get("stagingDir")
@@ -159,7 +129,7 @@ def process(self, instance):
# Fill tags and new families
tags = []
- if family_lowered in ("review", "renderlayer"):
+ if family_lowered in ("review", "renderlayer", "renderscene"):
tags.append("review")
# Sequence of one frame
@@ -167,7 +137,7 @@ def process(self, instance):
if single_file:
repre_files = repre_files[0]
- # Extension is harcoded
+ # Extension is hardcoded
# - changing extension would require change code
new_repre = {
"name": "png",
@@ -185,7 +155,7 @@ def process(self, instance):
instance.data["representations"].append(new_repre)
- if family_lowered in ("renderpass", "renderlayer"):
+ if family_lowered in ("renderpass", "renderlayer", "renderscene"):
# Change family to render
instance.data["family"] = "render"
@@ -234,7 +204,7 @@ def render_review(
scene_bg_color (list): Bg color set in scene. Result of george
script command `tv_background`.
- Retruns:
+ Returns:
tuple: With 2 items first is list of filenames second is path to
thumbnail.
"""
@@ -310,7 +280,7 @@ def render(self, output_dir, mark_in, mark_out, layers):
mark_out (int): On which frame index export will end.
layers (list): List of layers to be exported.
- Retruns:
+ Returns:
tuple: With 2 items first is list of filenames second is path to
thumbnail.
"""
diff --git a/openpype/hosts/tvpaint/plugins/publish/help/validate_asset_name.xml b/openpype/hosts/tvpaint/plugins/publish/help/validate_asset_name.xml
new file mode 100644
index 00000000000..33a9ca42478
--- /dev/null
+++ b/openpype/hosts/tvpaint/plugins/publish/help/validate_asset_name.xml
@@ -0,0 +1,22 @@
+
+
+
+Subset context
+## Invalid subset context
+
+Context of the given subset doesn't match your current scene.
+
+### How to repair?
+
+Yout can fix this with "Repair" button on the right. This will use '{expected_asset}' asset name and overwrite '{found_asset}' asset name in scene metadata.
+
+After that restart publishing with Reload button.
+
+
+### How could this happen?
+
+The subset was created in different scene with different context
+or the scene file was copy pasted from different context.
+
+
+
diff --git a/openpype/hosts/tvpaint/plugins/publish/help/validate_duplicated_layer_names.xml b/openpype/hosts/tvpaint/plugins/publish/help/validate_duplicated_layer_names.xml
new file mode 100644
index 00000000000..5d798544c0e
--- /dev/null
+++ b/openpype/hosts/tvpaint/plugins/publish/help/validate_duplicated_layer_names.xml
@@ -0,0 +1,22 @@
+
+
+
+Layer names
+## Duplicated layer names
+
+Can't determine which layers should be published because there are duplicated layer names in the scene.
+
+### Duplicated layer names
+
+{layer_names}
+
+*Check layer names for all subsets in list on left side.*
+
+### How to repair?
+
+Hide/rename/remove layers that should not be published.
+
+If all of them should be published then you have duplicated subset names in the scene. In that case you have to recrete them and use different variant name.
+
+
+
diff --git a/openpype/hosts/tvpaint/plugins/publish/help/validate_layers_visibility.xml b/openpype/hosts/tvpaint/plugins/publish/help/validate_layers_visibility.xml
new file mode 100644
index 00000000000..e7be7358881
--- /dev/null
+++ b/openpype/hosts/tvpaint/plugins/publish/help/validate_layers_visibility.xml
@@ -0,0 +1,20 @@
+
+
+
+Layers visiblity
+## All layers are not visible
+
+Layers visibility was changed during publishing which caused that all layers for subset "{instance_name}" are hidden.
+
+### Layer names for **{instance_name}**
+
+{layer_names}
+
+*Check layer names for all subsets in the list on the left side.*
+
+### How to repair?
+
+Reset publishing and do not change visibility of layers after hitting publish button.
+
+
+
diff --git a/openpype/hosts/tvpaint/plugins/publish/help/validate_marks.xml b/openpype/hosts/tvpaint/plugins/publish/help/validate_marks.xml
new file mode 100644
index 00000000000..f0e01ebaa78
--- /dev/null
+++ b/openpype/hosts/tvpaint/plugins/publish/help/validate_marks.xml
@@ -0,0 +1,21 @@
+
+
+
+Frame range
+## Invalid render frame range
+
+Scene frame range which will be rendered is defined by MarkIn and MarkOut. Expected frame range is {expected_frame_range} and current frame range is {current_frame_range}.
+
+It is also required that MarkIn and MarkOut are enabled in the scene. Their color is highlighted on timeline when are enabled.
+
+- MarkIn is {mark_in_enable_state}
+- MarkOut is {mark_out_enable_state}
+
+### How to repair?
+
+Yout can fix this with "Repair" button on the right. That will change MarkOut to {expected_mark_out}.
+
+Or you can manually modify MarkIn and MarkOut in the scene timeline.
+
+
+
diff --git a/openpype/hosts/tvpaint/plugins/publish/help/validate_missing_layer_names.xml b/openpype/hosts/tvpaint/plugins/publish/help/validate_missing_layer_names.xml
new file mode 100644
index 00000000000..e96e7c50444
--- /dev/null
+++ b/openpype/hosts/tvpaint/plugins/publish/help/validate_missing_layer_names.xml
@@ -0,0 +1,18 @@
+
+
+
+Missing layers
+## Missing layers for render pass
+
+Render pass subset "{instance_name}" has stored layer names that belong to it's rendering scope but layers were not found in scene.
+
+### Missing layer names
+
+{layer_names}
+
+### How to repair?
+
+Find layers that belong to subset {instance_name} and rename them back to expected layer names or remove the subset and create new with right layers.
+
+
+
diff --git a/openpype/hosts/tvpaint/plugins/publish/help/validate_render_pass_group.xml b/openpype/hosts/tvpaint/plugins/publish/help/validate_render_pass_group.xml
new file mode 100644
index 00000000000..df7bdf36e50
--- /dev/null
+++ b/openpype/hosts/tvpaint/plugins/publish/help/validate_render_pass_group.xml
@@ -0,0 +1,14 @@
+
+
+
+Render pass group
+## Invalid group of Render Pass layers
+
+Layers of Render Pass {instance_name} belong to Render Group which is defined by TVPaint color group {expected_group}. But the layers are not in the group.
+
+### How to repair?
+
+Change the color group to {expected_group} on layers {layer_names}.
+
+
+
diff --git a/openpype/hosts/tvpaint/plugins/publish/help/validate_scene_settings.xml b/openpype/hosts/tvpaint/plugins/publish/help/validate_scene_settings.xml
new file mode 100644
index 00000000000..f741c714562
--- /dev/null
+++ b/openpype/hosts/tvpaint/plugins/publish/help/validate_scene_settings.xml
@@ -0,0 +1,26 @@
+
+
+
+Scene settings
+## Invalid scene settings
+
+Scene settings do not match to expected values.
+
+**FPS**
+- Expected value: {expected_fps}
+- Current value: {current_fps}
+
+**Resolution**
+- Expected value: {expected_width}x{expected_height}
+- Current value: {current_width}x{current_height}
+
+**Pixel ratio**
+- Expected value: {expected_pixel_ratio}
+- Current value: {current_pixel_ratio}
+
+### How to repair?
+
+FPS and Pixel ratio can be modified in scene setting. Wrong resolution can be fixed with changing resolution of scene but due to TVPaint limitations it is possible that you will need to create new scene.
+
+
+
diff --git a/openpype/hosts/tvpaint/plugins/publish/help/validate_start_frame.xml b/openpype/hosts/tvpaint/plugins/publish/help/validate_start_frame.xml
new file mode 100644
index 00000000000..9052abf66c3
--- /dev/null
+++ b/openpype/hosts/tvpaint/plugins/publish/help/validate_start_frame.xml
@@ -0,0 +1,14 @@
+
+
+
+First frame
+## MarkIn is not set to 0
+
+MarkIn in your scene must start from 0 fram index but MarkIn is set to {current_start_frame}.
+
+### How to repair?
+
+You can modify MarkIn manually or hit the "Repair" button on the right which will change MarkIn to 0 (does not change MarkOut).
+
+
+
diff --git a/openpype/hosts/tvpaint/plugins/publish/help/validate_workfile_metadata.xml b/openpype/hosts/tvpaint/plugins/publish/help/validate_workfile_metadata.xml
new file mode 100644
index 00000000000..7397f6ef0b1
--- /dev/null
+++ b/openpype/hosts/tvpaint/plugins/publish/help/validate_workfile_metadata.xml
@@ -0,0 +1,19 @@
+
+
+
+Missing metadata
+## Your scene miss context metadata
+
+Your scene does not contain metadata about {missing_metadata}.
+
+### How to repair?
+
+Resave the scene using Workfiles tool or hit the "Repair" button on the right.
+
+
+### How this could happend?
+
+You're using scene file that was not created using Workfiles tool.
+
+
+
diff --git a/openpype/hosts/tvpaint/plugins/publish/help/validate_workfile_project_name.xml b/openpype/hosts/tvpaint/plugins/publish/help/validate_workfile_project_name.xml
new file mode 100644
index 00000000000..c4ffafc8b5f
--- /dev/null
+++ b/openpype/hosts/tvpaint/plugins/publish/help/validate_workfile_project_name.xml
@@ -0,0 +1,24 @@
+
+
+
+Project name
+## Your scene is from different project
+
+It is not possible to publish into project "{workfile_project_name}" when TVPaint was opened with project "{env_project_name}" in context.
+
+### How to repair?
+
+If the workfile belongs to project "{env_project_name}" then use Workfiles tool to resave it.
+
+Otherwise close TVPaint and launch it again from project you want to publish in.
+
+
+### How this could happend?
+
+You've opened workfile from different project. You've opened TVPaint on a task from "{env_project_name}" then you've opened TVPaint again on task from "{workfile_project_name}" without closing the TVPaint. Because TVPaint can run only once the project didn't change.
+
+### Why it is important?
+Because project may affect how TVPaint works or change publishing behavior it is dangerous to allow change project context in many ways. For example publishing will not run as expected.
+
+
+
diff --git a/openpype/hosts/tvpaint/plugins/publish/increment_workfile_version.py b/openpype/hosts/tvpaint/plugins/publish/increment_workfile_version.py
index a96a8e3d5de..24d6558168b 100644
--- a/openpype/hosts/tvpaint/plugins/publish/increment_workfile_version.py
+++ b/openpype/hosts/tvpaint/plugins/publish/increment_workfile_version.py
@@ -1,7 +1,7 @@
import pyblish.api
-from avalon.tvpaint import workio
from openpype.api import version_up
+from openpype.hosts.tvpaint.api import workio
class IncrementWorkfileVersion(pyblish.api.ContextPlugin):
@@ -15,7 +15,7 @@ class IncrementWorkfileVersion(pyblish.api.ContextPlugin):
def process(self, context):
assert all(result["success"] for result in context.data["results"]), (
- "Publishing not succesfull so version is not increased.")
+ "Publishing not successful so version is not increased.")
path = context.data["currentFile"]
workio.save_file(version_up(path))
diff --git a/openpype/hosts/tvpaint/plugins/publish/validate_asset_name.py b/openpype/hosts/tvpaint/plugins/publish/validate_asset_name.py
index 4ce8d5347de..70816f9f186 100644
--- a/openpype/hosts/tvpaint/plugins/publish/validate_asset_name.py
+++ b/openpype/hosts/tvpaint/plugins/publish/validate_asset_name.py
@@ -1,5 +1,6 @@
import pyblish.api
-from avalon.tvpaint import pipeline
+from openpype.pipeline import PublishXmlValidationError
+from openpype.hosts.tvpaint.api import pipeline
class FixAssetNames(pyblish.api.Action):
@@ -27,7 +28,7 @@ def process(self, context, plugin):
pipeline._write_instances(new_instance_items)
-class ValidateMissingLayers(pyblish.api.ContextPlugin):
+class ValidateAssetNames(pyblish.api.ContextPlugin):
"""Validate assset name present on instance.
Asset name on instance should be the same as context's.
@@ -48,8 +49,18 @@ def process(self, context):
instance_label = (
instance.data.get("label") or instance.data["name"]
)
- raise AssertionError((
- "Different asset name on instance then context's."
- " Instance \"{}\" has asset name: \"{}\""
- " Context asset name is: \"{}\""
- ).format(instance_label, asset_name, context_asset_name))
+
+ raise PublishXmlValidationError(
+ self,
+ (
+ "Different asset name on instance then context's."
+ " Instance \"{}\" has asset name: \"{}\""
+ " Context asset name is: \"{}\""
+ ).format(
+ instance_label, asset_name, context_asset_name
+ ),
+ formatting_data={
+ "expected_asset": context_asset_name,
+ "found_asset": asset_name
+ }
+ )
diff --git a/openpype/hosts/tvpaint/plugins/publish/validate_duplicated_layer_names.py b/openpype/hosts/tvpaint/plugins/publish/validate_duplicated_layer_names.py
index efccf19ef93..9f61bdbcd09 100644
--- a/openpype/hosts/tvpaint/plugins/publish/validate_duplicated_layer_names.py
+++ b/openpype/hosts/tvpaint/plugins/publish/validate_duplicated_layer_names.py
@@ -1,4 +1,5 @@
import pyblish.api
+from openpype.pipeline import PublishXmlValidationError
class ValidateLayersGroup(pyblish.api.InstancePlugin):
@@ -30,14 +31,20 @@ def process(self, instance):
"\"{}\"".format(layer_name)
for layer_name in duplicated_layer_names
])
-
- # Raise an error
- raise AssertionError(
+ detail_lines = [
+ "- {}".format(layer_name)
+ for layer_name in set(duplicated_layer_names)
+ ]
+ raise PublishXmlValidationError(
+ self,
(
"Layers have duplicated names for instance {}."
# Description what's wrong
" There are layers with same name and one of them is marked"
" for publishing so it is not possible to know which should"
" be published. Please look for layers with names: {}"
- ).format(instance.data["label"], layers_msg)
+ ).format(instance.data["label"], layers_msg),
+ formatting_data={
+ "layer_names": " before
+#include
+#endif
+
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+
+#include "plugdllx.h"
+
+#include
+
+#include
+#include
+
+#include "json.hpp"
+#include "jsonrpcpp.hpp"
+
+
+// All functions not exported should be static.
+// All global variables should be static.
+
+// mReq Identification of the requester. (=0 closed, !=0 requester ID)
+static struct {
+ bool firstParams;
+ DWORD mReq;
+ void* mLocalFile;
+ PIFilter *current_filter;
+ // Id counter for client requests
+ int client_request_id;
+ // There are new menu items
+ bool newMenuItems;
+ // Menu item definitions received from connection
+ nlohmann::json menuItems;
+ // Menu items used in requester by their ID
+ nlohmann::json menuItemsById;
+ std::list menuItemsIds;
+ // Messages from server before processing.
+ // - messages can't be process at the moment of receive as client is running in thread
+ std::queue messages;
+ // Responses to requests mapped by request id
+ std::map responses;
+
+} Data = {
+ true,
+ 0,
+ nullptr,
+ nullptr,
+ 1,
+ false,
+ nlohmann::json::object(),
+ nlohmann::json::object()
+};
+
+// Json rpc 2.0 parser - for handling messages and callbacks
+jsonrpcpp::Parser parser;
+typedef websocketpp::client client;
+
+
+class connection_metadata {
+private:
+ websocketpp::connection_hdl m_hdl;
+ client *m_endpoint;
+ std::string m_status;
+public:
+ typedef websocketpp::lib::shared_ptr ptr;
+
+ connection_metadata(websocketpp::connection_hdl hdl, client *endpoint)
+ : m_hdl(hdl), m_status("Connecting") {
+ m_endpoint = endpoint;
+ }
+
+ void on_open(client *c, websocketpp::connection_hdl hdl) {
+ m_status = "Open";
+ }
+
+ void on_fail(client *c, websocketpp::connection_hdl hdl) {
+ m_status = "Failed";
+ }
+
+ void on_close(client *c, websocketpp::connection_hdl hdl) {
+ m_status = "Closed";
+ }
+
+ void on_message(websocketpp::connection_hdl, client::message_ptr msg) {
+ std::string json_str;
+ if (msg->get_opcode() == websocketpp::frame::opcode::text) {
+ json_str = msg->get_payload();
+ } else {
+ json_str = websocketpp::utility::to_hex(msg->get_payload());
+ }
+ process_message(json_str);
+ }
+
+ void process_message(std::string msg) {
+ std::cout << "--> " << msg << "\n";
+ try {
+ jsonrpcpp::entity_ptr entity = parser.do_parse(msg);
+ if (!entity) {
+ // Return error code?
+
+ } else if (entity->is_response()) {
+ jsonrpcpp::Response response = jsonrpcpp::Response(entity->to_json());
+ Data.responses[response.id().int_id()] = response;
+
+ } else if (entity->is_request() || entity->is_notification()) {
+ Data.messages.push(msg);
+ }
+ }
+ catch (const jsonrpcpp::RequestException &e) {
+ std::string message = e.to_json().dump();
+ std::cout << "<-- " << e.to_json().dump() << "\n";
+ send(message);
+ }
+ catch (const jsonrpcpp::ParseErrorException &e) {
+ std::string message = e.to_json().dump();
+ std::cout << "<-- " << message << "\n";
+ send(message);
+ }
+ catch (const jsonrpcpp::RpcException &e) {
+ std::cerr << "RpcException: " << e.what() << "\n";
+ std::string message = jsonrpcpp::ParseErrorException(e.what()).to_json().dump();
+ std::cout << "<-- " << message << "\n";
+ send(message);
+ }
+ catch (const std::exception &e) {
+ std::cerr << "Exception: " << e.what() << "\n";
+ }
+ }
+
+ void send(std::string message) {
+ if (get_status() != "Open") {
+ return;
+ }
+ websocketpp::lib::error_code ec;
+
+ m_endpoint->send(m_hdl, message, websocketpp::frame::opcode::text, ec);
+ if (ec) {
+ std::cout << "> Error sending message: " << ec.message() << std::endl;
+ return;
+ }
+ }
+
+ void send_notification(jsonrpcpp::Notification *notification) {
+ send(notification->to_json().dump());
+ }
+
+ void send_response(jsonrpcpp::Response *response) {
+ send(response->to_json().dump());
+ }
+
+ void send_request(jsonrpcpp::Request *request) {
+ send(request->to_json().dump());
+ }
+
+ websocketpp::connection_hdl get_hdl() const {
+ return m_hdl;
+ }
+
+ std::string get_status() const {
+ return m_status;
+ }
+};
+
+
+class websocket_endpoint {
+private:
+ client m_endpoint;
+ connection_metadata::ptr client_metadata;
+ websocketpp::lib::shared_ptr m_thread;
+ bool thread_is_running = false;
+
+public:
+ websocket_endpoint() {
+ m_endpoint.clear_access_channels(websocketpp::log::alevel::all);
+ m_endpoint.clear_error_channels(websocketpp::log::elevel::all);
+ }
+
+ ~websocket_endpoint() {
+ close_connection();
+ }
+
+ void close_connection() {
+ m_endpoint.stop_perpetual();
+ if (connected())
+ {
+ // Close client
+ close(websocketpp::close::status::normal, "");
+ }
+ if (thread_is_running) {
+ // Join thread
+ m_thread->join();
+ thread_is_running = false;
+ }
+ }
+
+ bool connected()
+ {
+ return (client_metadata && client_metadata->get_status() == "Open");
+ }
+ int connect(std::string const &uri) {
+ if (client_metadata && client_metadata->get_status() == "Open") {
+ std::cout << "> Already connected" << std::endl;
+ return 0;
+ }
+
+ m_endpoint.init_asio();
+ m_endpoint.start_perpetual();
+
+ m_thread.reset(new websocketpp::lib::thread(&client::run, &m_endpoint));
+ thread_is_running = true;
+
+ websocketpp::lib::error_code ec;
+
+ client::connection_ptr con = m_endpoint.get_connection(uri, ec);
+
+ if (ec) {
+ std::cout << "> Connect initialization error: " << ec.message() << std::endl;
+ return -1;
+ }
+
+ client_metadata = websocketpp::lib::make_shared(con->get_handle(), &m_endpoint);
+
+ con->set_open_handler(websocketpp::lib::bind(
+ &connection_metadata::on_open,
+ client_metadata,
+ &m_endpoint,
+ websocketpp::lib::placeholders::_1
+ ));
+ con->set_fail_handler(websocketpp::lib::bind(
+ &connection_metadata::on_fail,
+ client_metadata,
+ &m_endpoint,
+ websocketpp::lib::placeholders::_1
+ ));
+ con->set_close_handler(websocketpp::lib::bind(
+ &connection_metadata::on_close,
+ client_metadata,
+ &m_endpoint,
+ websocketpp::lib::placeholders::_1
+ ));
+ con->set_message_handler(websocketpp::lib::bind(
+ &connection_metadata::on_message,
+ client_metadata,
+ websocketpp::lib::placeholders::_1,
+ websocketpp::lib::placeholders::_2
+ ));
+
+ m_endpoint.connect(con);
+
+ return 1;
+ }
+
+ void close(websocketpp::close::status::value code, std::string reason) {
+ if (!client_metadata || client_metadata->get_status() != "Open") {
+ std::cout << "> Not connected yet" << std::endl;
+ return;
+ }
+
+ websocketpp::lib::error_code ec;
+
+ m_endpoint.close(client_metadata->get_hdl(), code, reason, ec);
+ if (ec) {
+ std::cout << "> Error initiating close: " << ec.message() << std::endl;
+ }
+ }
+
+ void send(std::string message) {
+ if (!client_metadata || client_metadata->get_status() != "Open") {
+ std::cout << "> Not connected yet" << std::endl;
+ return;
+ }
+
+ client_metadata->send(message);
+ }
+
+ void send_notification(jsonrpcpp::Notification *notification) {
+ client_metadata->send_notification(notification);
+ }
+
+ void send_response(jsonrpcpp::Response *response) {
+ client_metadata->send(response->to_json().dump());
+ }
+
+ void send_response(std::shared_ptr response) {
+ client_metadata->send(response->to_json().dump());
+ }
+
+ void send_request(jsonrpcpp::Request *request) {
+ client_metadata->send_request(request);
+ }
+};
+
+class Communicator {
+private:
+ // URL to websocket server
+ std::string websocket_url;
+ // Should be avalon plugin available?
+ // - this may change during processing if websocketet url is not set or server is down
+ bool use_avalon;
+public:
+ Communicator();
+ websocket_endpoint endpoint;
+ bool is_connected();
+ bool is_usable();
+ void connect();
+ void process_requests();
+ jsonrpcpp::Response call_method(std::string method_name, nlohmann::json params);
+ void call_notification(std::string method_name, nlohmann::json params);
+};
+
+Communicator::Communicator() {
+ // URL to websocket server
+ websocket_url = std::getenv("WEBSOCKET_URL");
+ // Should be avalon plugin available?
+ // - this may change during processing if websocketet url is not set or server is down
+ if (websocket_url == "") {
+ use_avalon = false;
+ } else {
+ use_avalon = true;
+ }
+}
+
+bool Communicator::is_connected(){
+ return endpoint.connected();
+}
+
+bool Communicator::is_usable(){
+ return use_avalon;
+}
+
+void Communicator::connect()
+{
+ if (!use_avalon) {
+ return;
+ }
+ int con_result;
+ con_result = endpoint.connect(websocket_url);
+ if (con_result == -1)
+ {
+ use_avalon = false;
+ } else {
+ use_avalon = true;
+ }
+}
+
+void Communicator::call_notification(std::string method_name, nlohmann::json params) {
+ if (!use_avalon || !is_connected()) {return;}
+
+ jsonrpcpp::Notification notification = {method_name, params};
+ endpoint.send_notification(¬ification);
+}
+
+jsonrpcpp::Response Communicator::call_method(std::string method_name, nlohmann::json params) {
+ jsonrpcpp::Response response;
+ if (!use_avalon || !is_connected())
+ {
+ return response;
+ }
+ int request_id = Data.client_request_id++;
+ jsonrpcpp::Request request = {request_id, method_name, params};
+ endpoint.send_request(&request);
+
+ bool found = false;
+ while (!found) {
+ std::map::iterator iter = Data.responses.find(request_id);
+ if (iter != Data.responses.end()) {
+ //element found == was found response
+ response = iter->second;
+ Data.responses.erase(request_id);
+ found = true;
+ } else {
+ std::this_thread::sleep_for(std::chrono::milliseconds(100));
+ }
+ }
+ return response;
+}
+
+void Communicator::process_requests() {
+ if (!use_avalon || !is_connected() || Data.messages.empty()) {return;}
+
+ std::string msg = Data.messages.front();
+ Data.messages.pop();
+ std::cout << "Parsing: " << msg << std::endl;
+ // TODO: add try->except block
+ auto response = parser.parse(msg);
+ if (response->is_response()) {
+ endpoint.send_response(response);
+ } else {
+ jsonrpcpp::request_ptr request = std::dynamic_pointer_cast(response);
+ jsonrpcpp::Error error("Method \"" + request->method() + "\" not found", -32601);
+ jsonrpcpp::Response _response(request->id(), error);
+ endpoint.send_response(&_response);
+ }
+}
+
+jsonrpcpp::response_ptr define_menu(const jsonrpcpp::Id &id, const jsonrpcpp::Parameter ¶ms) {
+ /* Define plugin menu.
+
+ Menu is defined with json with "title" and "menu_items".
+ Each item in "menu_items" must have keys:
+ - "callback" - callback called with RPC when button is clicked
+ - "label" - label of button
+ - "help" - tooltip of button
+ ```
+ {
+ "title": "< Menu title>",
+ "menu_items": [
+ {
+ "callback": "workfiles_tool",
+ "label": "Workfiles",
+ "help": "Open workfiles tool"
+ },
+ ...
+ ]
+ }
+ ```
+ */
+ Data.menuItems = params.to_json()[0];
+ Data.newMenuItems = true;
+
+ std::string output;
+
+ return std::make_shared(id, output);
+}
+
+jsonrpcpp::response_ptr execute_george(const jsonrpcpp::Id &id, const jsonrpcpp::Parameter ¶ms) {
+ const char *george_script;
+ char cmd_output[1024] = {0};
+ char empty_char = {0};
+ std::string std_george_script;
+ std::string output;
+
+ nlohmann::json json_params = params.to_json();
+ std_george_script = json_params[0];
+ george_script = std_george_script.c_str();
+
+ // Result of `TVSendCmd` is int with length of output string
+ TVSendCmd(Data.current_filter, george_script, cmd_output);
+
+ for (int i = 0; i < sizeof(cmd_output); i++)
+ {
+ if (cmd_output[i] == empty_char){
+ break;
+ }
+ output += cmd_output[i];
+ }
+ return std::make_shared(id, output);
+}
+
+void register_callbacks(){
+ parser.register_request_callback("define_menu", define_menu);
+ parser.register_request_callback("execute_george", execute_george);
+}
+
+Communicator communication;
+
+////////////////////////////////////////////////////////////////////////////////////////
+
+static char* GetLocalString( PIFilter* iFilter, int iNum, char* iDefault )
+{
+ char* str;
+
+ if( Data.mLocalFile == NULL )
+ return iDefault;
+
+ str = TVGetLocalString( iFilter, Data.mLocalFile, iNum );
+ if( str == NULL || strlen( str ) == 0 )
+ return iDefault;
+
+ return str;
+}
+
+/**************************************************************************************/
+// Localisation
+
+// numbers (like 10011) are IDs in the localized file.
+// strings are the default values to use when the ID is not found
+// in the localized file (or the localized file doesn't exist).
+std::string label_from_evn()
+{
+ std::string _plugin_label = "Avalon";
+ if (std::getenv("AVALON_LABEL") && std::getenv("AVALON_LABEL") != "")
+ {
+ _plugin_label = std::getenv("AVALON_LABEL");
+ }
+ return _plugin_label;
+}
+std::string plugin_label = label_from_evn();
+
+#define TXT_REQUESTER GetLocalString( iFilter, 100, "OpenPype Tools" )
+
+#define TXT_REQUESTER_ERROR GetLocalString( iFilter, 30001, "Can't Open Requester !" )
+
+////////////////////////////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////////////////////////////
+
+// The functions directly called by Aura through the plugin interface
+
+
+
+/**************************************************************************************/
+// "About" function.
+
+
+void FAR PASCAL PI_About( PIFilter* iFilter )
+{
+ char text[256];
+
+ sprintf( text, "%s %d,%d", iFilter->PIName, iFilter->PIVersion, iFilter->PIRevision );
+
+ // Just open a warning popup with the filter name and version.
+ // You can open a much nicer requester if you want.
+ TVWarning( iFilter, text );
+}
+
+
+/**************************************************************************************/
+// Function called at Aura startup, when the filter is loaded.
+// Should do as little as possible to keep Aura's startup time small.
+
+int FAR PASCAL PI_Open( PIFilter* iFilter )
+{
+ Data.current_filter = iFilter;
+ char tmp[256];
+
+ strcpy( iFilter->PIName, plugin_label.c_str() );
+ iFilter->PIVersion = 1;
+ iFilter->PIRevision = 0;
+
+ // If this plugin was the one open at Aura shutdown, re-open it
+ TVReadUserString( iFilter, iFilter->PIName, "Open", tmp, "0", 255 );
+ if( atoi( tmp ) )
+ {
+ PI_Parameters( iFilter, NULL ); // NULL as iArg means "open the requester"
+ }
+
+ communication.connect();
+ register_callbacks();
+ return 1; // OK
+}
+
+
+/**************************************************************************************/
+// Aura shutdown: we make all the necessary cleanup
+
+void FAR PASCAL PI_Close( PIFilter* iFilter )
+{
+ if( Data.mLocalFile )
+ {
+ TVCloseLocalFile( iFilter, Data.mLocalFile );
+ }
+ if( Data.mReq )
+ {
+ TVCloseReq( iFilter, Data.mReq );
+ }
+ communication.endpoint.close_connection();
+}
+
+
+/**************************************************************************************/
+// we have something to do !
+
+int FAR PASCAL PI_Parameters( PIFilter* iFilter, char* iArg )
+{
+ if( !iArg )
+ {
+
+ // If the requester is not open, we open it.
+ if( Data.mReq == 0)
+ {
+ // Create empty requester because menu items are defined with
+ // `define_menu` callback
+ DWORD req = TVOpenFilterReqEx(
+ iFilter,
+ 185,
+ 20,
+ NULL,
+ NULL,
+ PIRF_STANDARD_REQ | PIRF_COLLAPSABLE_REQ,
+ FILTERREQ_NO_TBAR
+ );
+ if( req == 0 )
+ {
+ TVWarning( iFilter, TXT_REQUESTER_ERROR );
+ return 0;
+ }
+
+
+ Data.mReq = req;
+ // This is a very simple requester, so we create it's content right here instead
+ // of waiting for the PICBREQ_OPEN message...
+ // Not recommended for more complex requesters. (see the other examples)
+
+ // Sets the title of the requester.
+ TVSetReqTitle( iFilter, Data.mReq, TXT_REQUESTER );
+ // Request to listen to ticks
+ TVGrabTicks(iFilter, req, PITICKS_FLAG_ON);
+ }
+ else
+ {
+ // If it is already open, we just put it on front of all other requesters.
+ TVReqToFront( iFilter, Data.mReq );
+ }
+ }
+
+ return 1;
+}
+
+
+int newMenuItemsProcess(PIFilter* iFilter) {
+ // Menu items defined with `define_menu` should be propagated.
+
+ // Change flag that there are new menu items (avoid infinite loop)
+ Data.newMenuItems = false;
+ // Skip if requester does not exists
+ if (Data.mReq == 0) {
+ return 0;
+ }
+ // Remove all previous menu items
+ for (int menu_id : Data.menuItemsIds)
+ {
+ TVRemoveButtonReq(iFilter, Data.mReq, menu_id);
+ }
+ // Clear caches
+ Data.menuItemsById.clear();
+ Data.menuItemsIds.clear();
+
+ // We use a variable to contains the vertical position of the buttons.
+ // Each time we create a button, we add its size to this variable.
+ // This makes it very easy to add/remove/displace buttons in a requester.
+ int x_pos = 9;
+ int y_pos = 5;
+
+ // Menu width
+ int menu_width = 185;
+ // Single menu item width
+ int btn_width = menu_width - 19;
+ // Single row height (btn height is 18)
+ int row_height = 20;
+ // Additional height to menu
+ int height_offset = 5;
+
+ // This is a very simple requester, so we create it's content right here instead
+ // of waiting for the PICBREQ_OPEN message...
+ // Not recommended for more complex requesters. (see the other examples)
+
+ const char *menu_title = TXT_REQUESTER;
+ if (Data.menuItems.contains("title"))
+ {
+ menu_title = Data.menuItems["title"].get()->c_str();
+ }
+ // Sets the title of the requester.
+ TVSetReqTitle( iFilter, Data.mReq, menu_title );
+
+ // Resize menu
+ // First get current position and sizes (we only need the position)
+ int current_x = 0;
+ int current_y = 0;
+ int current_width = 0;
+ int current_height = 0;
+ TVInfoReq(iFilter, Data.mReq, ¤t_x, ¤t_y, ¤t_width, ¤t_height);
+
+ // Calculate new height
+ int menu_height = (row_height * Data.menuItems["menu_items"].size()) + height_offset;
+ // Resize
+ TVResizeReq(iFilter, Data.mReq, current_x, current_y, menu_width, menu_height);
+
+ // Add menu items
+ int item_counter = 1;
+ for (auto& item : Data.menuItems["menu_items"].items())
+ {
+ int item_id = item_counter * 10;
+ item_counter ++;
+ std::string item_id_str = std::to_string(item_id);
+ nlohmann::json item_data = item.value();
+ const char *item_label = item_data["label"].get()->c_str();
+ const char *help_text = item_data["help"].get()->c_str();
+ std::string item_callback = item_data["callback"].get();
+ TVAddButtonReq(iFilter, Data.mReq, x_pos, y_pos, btn_width, 0, item_id, PIRBF_BUTTON_NORMAL|PIRBF_BUTTON_ACTION, item_label);
+ TVSetButtonInfoText( iFilter, Data.mReq, item_id, help_text );
+ y_pos += row_height;
+
+ Data.menuItemsById[std::to_string(item_id)] = item_callback;
+ Data.menuItemsIds.push_back(item_id);
+ }
+
+ return 1;
+}
+/**************************************************************************************/
+// something happened that needs our attention.
+// Global variable where current button up data are stored
+std::string button_up_item_id_str;
+int FAR PASCAL PI_Msg( PIFilter* iFilter, INTPTR iEvent, INTPTR iReq, INTPTR* iArgs )
+{
+ Data.current_filter = iFilter;
+ // what did happen ?
+ switch( iEvent )
+ {
+ // The user just 'clicked' on a normal button
+ case PICBREQ_BUTTON_UP:
+ button_up_item_id_str = std::to_string(iArgs[0]);
+ if (Data.menuItemsById.contains(button_up_item_id_str))
+ {
+ std::string callback_name = Data.menuItemsById[button_up_item_id_str].get();
+ communication.call_method(callback_name, nlohmann::json::array());
+ }
+ TVExecute( iFilter );
+ break;
+
+ // The requester was just closed.
+ case PICBREQ_CLOSE:
+ // requester doesn't exists anymore
+ Data.mReq = 0;
+
+ char tmp[256];
+ // Save the requester state (opened or closed)
+ // iArgs[4] contains a flag which tells us if the requester
+ // has been closed by the user (flag=0) or by Aura's shutdown (flag=1).
+ // If it was by Aura's shutdown, that means this requester was the
+ // last one open, so we should reopen this one the next time Aura
+ // is started. Else we won't open it next time.
+ sprintf( tmp, "%d", (int)(iArgs[4]) );
+
+ // Save it in Aura's init file.
+ TVWriteUserString( iFilter, iFilter->PIName, "Open", tmp );
+ break;
+
+ case PICBREQ_TICKS:
+ if (Data.newMenuItems)
+ {
+ newMenuItemsProcess(iFilter);
+ }
+ communication.process_requests();
+ }
+
+ return 1;
+}
+
+
+/**************************************************************************************/
+// Start of the 'execution' of the filter for a new sequence.
+// - iNumImages contains the total number of frames to be processed.
+// Here you should allocate memory that is used for all frames,
+// and precompute all the stuff that doesn't change from frame to frame.
+
+
+int FAR PASCAL PI_SequenceStart( PIFilter* iFilter, int iNumImages )
+{
+ // In this simple example we don't have anything to allocate/precompute.
+
+ // 1 means 'continue', 0 means 'error, abort' (like 'not enough memory')
+ return 1;
+}
+
+
+// Here you should cleanup what you've done in PI_SequenceStart
+
+void FAR PASCAL PI_SequenceFinish( PIFilter* iFilter )
+{}
+
+
+/**************************************************************************************/
+// This is called before each frame.
+// Here you should allocate memory and precompute all the stuff you can.
+
+int FAR PASCAL PI_Start( PIFilter* iFilter, double iPos, double iSize )
+{
+ return 1;
+}
+
+
+void FAR PASCAL PI_Finish( PIFilter* iFilter )
+{
+ // nothing special to cleanup
+}
+
+
+/**************************************************************************************/
+// 'Execution' of the filter.
+int FAR PASCAL PI_Work( PIFilter* iFilter )
+{
+ return 1;
+}
diff --git a/openpype/hosts/tvpaint/tvpaint_plugin/plugin_code/library.def b/openpype/hosts/tvpaint/tvpaint_plugin/plugin_code/library.def
new file mode 100644
index 00000000000..882f2b4719c
--- /dev/null
+++ b/openpype/hosts/tvpaint/tvpaint_plugin/plugin_code/library.def
@@ -0,0 +1,10 @@
+LIBRARY Avalonplugin
+EXPORTS
+ PI_Msg
+ PI_Open
+ PI_About
+ PI_Parameters
+ PI_Start
+ PI_Work
+ PI_Finish
+ PI_Close
diff --git a/openpype/hosts/tvpaint/tvpaint_plugin/plugin_files/windows_x64/plugin/OpenPypePlugin.dll b/openpype/hosts/tvpaint/tvpaint_plugin/plugin_files/windows_x64/plugin/OpenPypePlugin.dll
new file mode 100644
index 00000000000..f7f5119ef34
Binary files /dev/null and b/openpype/hosts/tvpaint/tvpaint_plugin/plugin_files/windows_x64/plugin/OpenPypePlugin.dll differ
diff --git a/openpype/hosts/tvpaint/tvpaint_plugin/plugin_files/windows_x86/plugin/OpenPypePlugin.dll b/openpype/hosts/tvpaint/tvpaint_plugin/plugin_files/windows_x86/plugin/OpenPypePlugin.dll
new file mode 100644
index 00000000000..f35e3ffe868
Binary files /dev/null and b/openpype/hosts/tvpaint/tvpaint_plugin/plugin_files/windows_x86/plugin/OpenPypePlugin.dll differ
diff --git a/openpype/hosts/tvpaint/worker/init_file.tvpp b/openpype/hosts/tvpaint/worker/init_file.tvpp
new file mode 100644
index 00000000000..22170b45bc7
Binary files /dev/null and b/openpype/hosts/tvpaint/worker/init_file.tvpp differ
diff --git a/openpype/hosts/tvpaint/worker/worker.py b/openpype/hosts/tvpaint/worker/worker.py
index 738656fa91a..9295c8afb47 100644
--- a/openpype/hosts/tvpaint/worker/worker.py
+++ b/openpype/hosts/tvpaint/worker/worker.py
@@ -1,8 +1,11 @@
+import os
import signal
import time
+import tempfile
+import shutil
import asyncio
-from avalon.tvpaint.communication_server import (
+from openpype.hosts.tvpaint.api.communication_server import (
BaseCommunicator,
CommunicationWrapper
)
@@ -36,8 +39,28 @@ def _start_webserver(self):
super()._start_webserver()
+ def _open_init_file(self):
+ """Open init TVPaint file.
+
+ File triggers dialog missing path to audio file which must be closed
+ once and is ignored for rest of running process.
+ """
+ current_dir = os.path.dirname(os.path.abspath(__file__))
+ init_filepath = os.path.join(current_dir, "init_file.tvpp")
+ with tempfile.NamedTemporaryFile(
+ mode="w", prefix="a_tvp_", suffix=".tvpp"
+ ) as tmp_file:
+ tmp_filepath = tmp_file.name.replace("\\", "/")
+
+ shutil.copy(init_filepath, tmp_filepath)
+ george_script = "tv_LoadProject '\"'\"{}\"'\"'".format(tmp_filepath)
+ self.execute_george_through_file(george_script)
+ self.execute_george("tv_projectclose")
+ os.remove(tmp_filepath)
+
def _on_client_connect(self, *args, **kwargs):
super()._on_client_connect(*args, **kwargs)
+ self._open_init_file()
# Register as "ready to work" worker
self._worker_connection.register_as_worker()
diff --git a/openpype/hosts/tvpaint/worker/worker_job.py b/openpype/hosts/tvpaint/worker/worker_job.py
index c3893b6f2ea..95c0a678bce 100644
--- a/openpype/hosts/tvpaint/worker/worker_job.py
+++ b/openpype/hosts/tvpaint/worker/worker_job.py
@@ -9,7 +9,7 @@
import six
-from openpype.api import PypeLogger
+from openpype.lib import Logger
from openpype.modules import ModulesManager
@@ -41,7 +41,7 @@ class BaseCommand:
Command also have id which is created on command creation.
The idea is that command is just a data container on sender side send
- througth server to a worker where is replicated one by one, executed and
+ through server to a worker where is replicated one by one, executed and
result sent back to sender through server.
"""
@abstractproperty
@@ -248,7 +248,7 @@ def from_existing(cls, data):
class CollectSceneData(BaseCommand):
- """Helper command which will collect all usefull info about workfile.
+ """Helper command which will collect all useful info about workfile.
Result is dictionary with all layers data, exposure frames by layer ids
pre/post behavior of layers by their ids, group information and scene data.
@@ -256,7 +256,7 @@ class CollectSceneData(BaseCommand):
name = "collect_scene_data"
def execute(self):
- from avalon.tvpaint.lib import (
+ from openpype.hosts.tvpaint.api.lib import (
get_layers_data,
get_groups_data,
get_layers_pre_post_behavior,
@@ -328,7 +328,7 @@ def job_queue_root(self):
def log(self):
"""Access to logger object."""
if self._log is None:
- self._log = PypeLogger.get_logger(self.__class__.__name__)
+ self._log = Logger.get_logger(self.__class__.__name__)
return self._log
@property
diff --git a/openpype/hosts/unreal/__init__.py b/openpype/hosts/unreal/__init__.py
index 1280442916a..42dd8f0ac47 100644
--- a/openpype/hosts/unreal/__init__.py
+++ b/openpype/hosts/unreal/__init__.py
@@ -1,18 +1,6 @@
-import os
+from .addon import UnrealAddon
-def add_implementation_envs(env, _app):
- """Modify environments to contain all required for implementation."""
- # Set AVALON_UNREAL_PLUGIN required for Unreal implementation
- unreal_plugin_path = os.path.join(
- os.environ["OPENPYPE_REPOS_ROOT"], "repos", "avalon-unreal-integration"
- )
- env["AVALON_UNREAL_PLUGIN"] = unreal_plugin_path
-
- # Set default environments if are not set via settings
- defaults = {
- "OPENPYPE_LOG_NO_COLORS": "True"
- }
- for key, value in defaults.items():
- if not env.get(key):
- env[key] = value
+__all__ = (
+ "UnrealAddon",
+)
diff --git a/openpype/hosts/unreal/addon.py b/openpype/hosts/unreal/addon.py
new file mode 100644
index 00000000000..16736214c52
--- /dev/null
+++ b/openpype/hosts/unreal/addon.py
@@ -0,0 +1,42 @@
+import os
+from openpype.modules import OpenPypeModule
+from openpype.modules.interfaces import IHostAddon
+
+UNREAL_ROOT_DIR = os.path.dirname(os.path.abspath(__file__))
+
+
+class UnrealAddon(OpenPypeModule, IHostAddon):
+ name = "unreal"
+ host_name = "unreal"
+
+ def initialize(self, module_settings):
+ self.enabled = True
+
+ def add_implementation_envs(self, env, app):
+ """Modify environments to contain all required for implementation."""
+ # Set OPENPYPE_UNREAL_PLUGIN required for Unreal implementation
+
+ ue_plugin = "UE_5.0" if app.name[:1] == "5" else "UE_4.7"
+ unreal_plugin_path = os.path.join(
+ UNREAL_ROOT_DIR, "integration", ue_plugin
+ )
+ if not env.get("OPENPYPE_UNREAL_PLUGIN"):
+ env["OPENPYPE_UNREAL_PLUGIN"] = unreal_plugin_path
+
+ # Set default environments if are not set via settings
+ defaults = {
+ "OPENPYPE_LOG_NO_COLORS": "True"
+ }
+ for key, value in defaults.items():
+ if not env.get(key):
+ env[key] = value
+
+ def get_launch_hook_paths(self, app):
+ if app.host_name != self.host_name:
+ return []
+ return [
+ os.path.join(UNREAL_ROOT_DIR, "hooks")
+ ]
+
+ def get_workfile_extensions(self):
+ return [".uproject"]
diff --git a/openpype/hosts/unreal/api/__init__.py b/openpype/hosts/unreal/api/__init__.py
index 38469e0ddba..870982f5f9a 100644
--- a/openpype/hosts/unreal/api/__init__.py
+++ b/openpype/hosts/unreal/api/__init__.py
@@ -1,45 +1,42 @@
-import os
-import logging
+# -*- coding: utf-8 -*-
+"""Unreal Editor OpenPype host API."""
-from avalon import api as avalon
-from pyblish import api as pyblish
-import openpype.hosts.unreal
+from .plugin import (
+ Loader,
+ Creator
+)
+from .pipeline import (
+ install,
+ uninstall,
+ ls,
+ publish,
+ containerise,
+ show_creator,
+ show_loader,
+ show_publisher,
+ show_manager,
+ show_experimental_tools,
+ show_tools_dialog,
+ show_tools_popup,
+ instantiate,
+ UnrealHost,
+)
-logger = logging.getLogger("openpype.hosts.unreal")
-
-HOST_DIR = os.path.dirname(os.path.abspath(openpype.hosts.unreal.__file__))
-PLUGINS_DIR = os.path.join(HOST_DIR, "plugins")
-PUBLISH_PATH = os.path.join(PLUGINS_DIR, "publish")
-LOAD_PATH = os.path.join(PLUGINS_DIR, "load")
-CREATE_PATH = os.path.join(PLUGINS_DIR, "create")
-INVENTORY_PATH = os.path.join(PLUGINS_DIR, "inventory")
-
-
-def install():
- """Install Unreal configuration for Avalon."""
- print("-=" * 40)
- logo = '''.
-.
- ____________
- / \\ __ \\
- \\ \\ \\/_\\ \\
- \\ \\ _____/ ______
- \\ \\ \\___// \\ \\
- \\ \\____\\ \\ \\_____\\
- \\/_____/ \\/______/ PYPE Club .
-.
-'''
- print(logo)
- print("installing OpenPype for Unreal ...")
- print("-=" * 40)
- logger.info("installing OpenPype for Unreal")
- pyblish.register_plugin_path(str(PUBLISH_PATH))
- avalon.register_plugin_path(avalon.Loader, str(LOAD_PATH))
- avalon.register_plugin_path(avalon.Creator, str(CREATE_PATH))
-
-
-def uninstall():
- """Uninstall Unreal configuration for Avalon."""
- pyblish.deregister_plugin_path(str(PUBLISH_PATH))
- avalon.deregister_plugin_path(avalon.Loader, str(LOAD_PATH))
- avalon.deregister_plugin_path(avalon.Creator, str(CREATE_PATH))
+__all__ = [
+ "install",
+ "uninstall",
+ "Creator",
+ "Loader",
+ "ls",
+ "publish",
+ "containerise",
+ "show_creator",
+ "show_loader",
+ "show_publisher",
+ "show_manager",
+ "show_experimental_tools",
+ "show_tools_dialog",
+ "show_tools_popup",
+ "instantiate",
+ "UnrealHost",
+]
diff --git a/openpype/hosts/unreal/api/helpers.py b/openpype/hosts/unreal/api/helpers.py
new file mode 100644
index 00000000000..0b6f07f52f8
--- /dev/null
+++ b/openpype/hosts/unreal/api/helpers.py
@@ -0,0 +1,44 @@
+# -*- coding: utf-8 -*-
+import unreal # noqa
+
+
+class OpenPypeUnrealException(Exception):
+ pass
+
+
+@unreal.uclass()
+class OpenPypeHelpers(unreal.OpenPypeLib):
+ """Class wrapping some useful functions for OpenPype.
+
+ This class is extending native BP class in OpenPype Integration Plugin.
+
+ """
+
+ @unreal.ufunction(params=[str, unreal.LinearColor, bool])
+ def set_folder_color(self, path: str, color: unreal.LinearColor) -> None:
+ """Set color on folder in Content Browser.
+
+ This method sets color on folder in Content Browser. Unfortunately
+ there is no way to refresh Content Browser so new color isn't applied
+ immediately. They are saved to config file and appears correctly
+ only after Editor is restarted.
+
+ Args:
+ path (str): Path to folder
+ color (:class:`unreal.LinearColor`): Color of the folder
+
+ Example:
+
+ OpenPypeHelpers().set_folder_color(
+ "/Game/Path", unreal.LinearColor(a=1.0, r=1.0, g=0.5, b=0)
+ )
+
+ Note:
+ This will take effect only after Editor is restarted. I couldn't
+ find a way to refresh it. Also this saves the color definition
+ into the project config, binding this path with color. So if you
+ delete this path and later re-create, it will set this color
+ again.
+
+ """
+ self.c_set_folder_color(path, color, False)
diff --git a/openpype/hosts/unreal/api/lib.py b/openpype/hosts/unreal/api/lib.py
deleted file mode 100644
index c0fafbb667e..00000000000
--- a/openpype/hosts/unreal/api/lib.py
+++ /dev/null
@@ -1,466 +0,0 @@
-# -*- coding: utf-8 -*-
-"""Unreal launching and project tools."""
-import sys
-import os
-import platform
-import json
-from distutils import dir_util
-import subprocess
-import re
-from pathlib import Path
-from collections import OrderedDict
-from openpype.api import get_project_settings
-
-
-def get_engine_versions(env=None):
- """Detect Unreal Engine versions.
-
- This will try to detect location and versions of installed Unreal Engine.
- Location can be overridden by `UNREAL_ENGINE_LOCATION` environment
- variable.
-
- Args:
- env (dict, optional): Environment to use.
-
- Returns:
- OrderedDict: dictionary with version as a key and dir as value.
- so the highest version is first.
-
- Example:
- >>> get_engine_versions()
- {
- "4.23": "C:/Epic Games/UE_4.23",
- "4.24": "C:/Epic Games/UE_4.24"
- }
-
- """
- env = env or os.environ
- engine_locations = {}
- try:
- root, dirs, _ = next(os.walk(env["UNREAL_ENGINE_LOCATION"]))
-
- for directory in dirs:
- if directory.startswith("UE"):
- try:
- ver = re.split(r"[-_]", directory)[1]
- except IndexError:
- continue
- engine_locations[ver] = os.path.join(root, directory)
- except KeyError:
- # environment variable not set
- pass
- except OSError:
- # specified directory doesn't exists
- pass
-
- # if we've got something, terminate auto-detection process
- if engine_locations:
- return OrderedDict(sorted(engine_locations.items()))
-
- # else kick in platform specific detection
- if platform.system().lower() == "windows":
- return OrderedDict(sorted(_win_get_engine_versions().items()))
- if platform.system().lower() == "linux":
- # on linux, there is no installation and getting Unreal Engine involves
- # git clone. So we'll probably depend on `UNREAL_ENGINE_LOCATION`.
- pass
- if platform.system().lower() == "darwin":
- return OrderedDict(sorted(_darwin_get_engine_version().items()))
-
- return OrderedDict()
-
-
-def get_editor_executable_path(engine_path: Path) -> Path:
- """Get UE4 Editor executable path."""
- ue4_path = engine_path / "Engine/Binaries"
- if platform.system().lower() == "windows":
- ue4_path /= "Win64/UE4Editor.exe"
-
- elif platform.system().lower() == "linux":
- ue4_path /= "Linux/UE4Editor"
-
- elif platform.system().lower() == "darwin":
- ue4_path /= "Mac/UE4Editor"
-
- return ue4_path
-
-
-def _win_get_engine_versions():
- """Get Unreal Engine versions on Windows.
-
- If engines are installed via Epic Games Launcher then there is:
- `%PROGRAMDATA%/Epic/UnrealEngineLauncher/LauncherInstalled.dat`
- This file is JSON file listing installed stuff, Unreal engines
- are marked with `"AppName" = "UE_X.XX"`` like `UE_4.24`
-
- Returns:
- dict: version as a key and path as a value.
-
- """
- install_json_path = os.path.join(
- os.getenv("PROGRAMDATA"),
- "Epic",
- "UnrealEngineLauncher",
- "LauncherInstalled.dat",
- )
-
- return _parse_launcher_locations(install_json_path)
-
-
-def _darwin_get_engine_version() -> dict:
- """Get Unreal Engine versions on MacOS.
-
- It works the same as on Windows, just JSON file location is different.
-
- Returns:
- dict: version as a key and path as a value.
-
- See Aslo:
- :func:`_win_get_engine_versions`.
-
- """
- install_json_path = os.path.join(
- os.getenv("HOME"),
- "Library",
- "Application Support",
- "Epic",
- "UnrealEngineLauncher",
- "LauncherInstalled.dat",
- )
-
- return _parse_launcher_locations(install_json_path)
-
-
-def _parse_launcher_locations(install_json_path: str) -> dict:
- """This will parse locations from json file.
-
- Args:
- install_json_path (str): Path to `LauncherInstalled.dat`.
-
- Returns:
- dict: with unreal engine versions as keys and
- paths to those engine installations as value.
-
- """
- engine_locations = {}
- if os.path.isfile(install_json_path):
- with open(install_json_path, "r") as ilf:
- try:
- install_data = json.load(ilf)
- except json.JSONDecodeError as e:
- raise Exception(
- "Invalid `LauncherInstalled.dat file. `"
- "Cannot determine Unreal Engine location."
- ) from e
-
- for installation in install_data.get("InstallationList", []):
- if installation.get("AppName").startswith("UE_"):
- ver = installation.get("AppName").split("_")[1]
- engine_locations[ver] = installation.get("InstallLocation")
-
- return engine_locations
-
-
-def create_unreal_project(project_name: str,
- ue_version: str,
- pr_dir: Path,
- engine_path: Path,
- dev_mode: bool = False,
- env: dict = None) -> None:
- """This will create `.uproject` file at specified location.
-
- As there is no way I know to create project via command line, this is
- easiest option. Unreal project file is basically JSON file. If we find
- `AVALON_UNREAL_PLUGIN` environment variable we assume this is location
- of Avalon Integration Plugin and we copy its content to project folder
- and enable this plugin.
-
- Args:
- project_name (str): Name of the project.
- ue_version (str): Unreal engine version (like 4.23).
- pr_dir (Path): Path to directory where project will be created.
- engine_path (Path): Path to Unreal Engine installation.
- dev_mode (bool, optional): Flag to trigger C++ style Unreal project
- needing Visual Studio and other tools to compile plugins from
- sources. This will trigger automatically if `Binaries`
- directory is not found in plugin folders as this indicates
- this is only source distribution of the plugin. Dev mode
- is also set by preset file `unreal/project_setup.json` in
- **OPENPYPE_CONFIG**.
- env (dict, optional): Environment to use. If not set, `os.environ`.
-
- Throws:
- NotImplementedError: For unsupported platforms.
-
- Returns:
- None
-
- """
- env = env or os.environ
- preset = get_project_settings(project_name)["unreal"]["project_setup"]
- ue_id = ".".join(ue_version.split(".")[:2])
- # get unreal engine identifier
- # -------------------------------------------------------------------------
- # FIXME (antirotor): As of 4.26 this is problem with UE4 built from
- # sources. In that case Engine ID is calculated per machine/user and not
- # from Engine files as this code then reads. This then prevents UE4
- # to directly open project as it will complain about project being
- # created in different UE4 version. When user convert such project
- # to his UE4 version, Engine ID is replaced in uproject file. If some
- # other user tries to open it, it will present him with similar error.
- ue4_modules = Path()
- if platform.system().lower() == "windows":
- ue4_modules = Path(os.path.join(engine_path, "Engine", "Binaries",
- "Win64", "UE4Editor.modules"))
-
- if platform.system().lower() == "linux":
- ue4_modules = Path(os.path.join(engine_path, "Engine", "Binaries",
- "Linux", "UE4Editor.modules"))
-
- if platform.system().lower() == "darwin":
- ue4_modules = Path(os.path.join(engine_path, "Engine", "Binaries",
- "Mac", "UE4Editor.modules"))
-
- if ue4_modules.exists():
- print("--- Loading Engine ID from modules file ...")
- with open(ue4_modules, "r") as mp:
- loaded_modules = json.load(mp)
-
- if loaded_modules.get("BuildId"):
- ue_id = "{" + loaded_modules.get("BuildId") + "}"
-
- plugins_path = None
- if os.path.isdir(env.get("AVALON_UNREAL_PLUGIN", "")):
- # copy plugin to correct path under project
- plugins_path = pr_dir / "Plugins"
- avalon_plugin_path = plugins_path / "Avalon"
- if not avalon_plugin_path.is_dir():
- avalon_plugin_path.mkdir(parents=True, exist_ok=True)
- dir_util._path_created = {}
- dir_util.copy_tree(os.environ.get("AVALON_UNREAL_PLUGIN"),
- avalon_plugin_path.as_posix())
-
- if not (avalon_plugin_path / "Binaries").is_dir() \
- or not (avalon_plugin_path / "Intermediate").is_dir():
- dev_mode = True
-
- # data for project file
- data = {
- "FileVersion": 3,
- "EngineAssociation": ue_id,
- "Category": "",
- "Description": "",
- "Plugins": [
- {"Name": "PythonScriptPlugin", "Enabled": True},
- {"Name": "EditorScriptingUtilities", "Enabled": True},
- {"Name": "SequencerScripting", "Enabled": True},
- {"Name": "Avalon", "Enabled": True}
- ]
- }
-
- if dev_mode or preset["dev_mode"]:
- # this will add project module and necessary source file to make it
- # C++ project and to (hopefully) make Unreal Editor to compile all
- # sources at start
-
- data["Modules"] = [{
- "Name": project_name,
- "Type": "Runtime",
- "LoadingPhase": "Default",
- "AdditionalDependencies": ["Engine"],
- }]
-
- # write project file
- project_file = pr_dir / f"{project_name}.uproject"
- with open(project_file, mode="w") as pf:
- json.dump(data, pf, indent=4)
-
- # ensure we have PySide2 installed in engine
- python_path = None
- if platform.system().lower() == "windows":
- python_path = engine_path / ("Engine/Binaries/ThirdParty/"
- "Python3/Win64/pythonw.exe")
-
- if platform.system().lower() == "linux":
- python_path = engine_path / ("Engine/Binaries/ThirdParty/"
- "Python3/Linux/bin/python3")
-
- if platform.system().lower() == "darwin":
- python_path = engine_path / ("Engine/Binaries/ThirdParty/"
- "Python3/Mac/bin/python3")
-
- if not python_path:
- raise NotImplementedError("Unsupported platform")
- if not python_path.exists():
- raise RuntimeError(f"Unreal Python not found at {python_path}")
- subprocess.run(
- [python_path.as_posix(), "-m", "pip", "install", "pyside2"])
-
- if dev_mode or preset["dev_mode"]:
- _prepare_cpp_project(project_file, engine_path)
-
-
-def _prepare_cpp_project(project_file: Path, engine_path: Path) -> None:
- """Prepare CPP Unreal Project.
-
- This function will add source files needed for project to be
- rebuild along with the avalon integration plugin.
-
- There seems not to be automated way to do it from command line.
- But there might be way to create at least those target and build files
- by some generator. This needs more research as manually writing
- those files is rather hackish. :skull_and_crossbones:
-
-
- Args:
- project_file (str): Path to .uproject file.
- engine_path (str): Path to unreal engine associated with project.
-
- """
- project_name = project_file.stem
- project_dir = project_file.parent
- targets_dir = project_dir / "Source"
- sources_dir = targets_dir / project_name
-
- sources_dir.mkdir(parents=True, exist_ok=True)
- (project_dir / "Content").mkdir(parents=True, exist_ok=True)
-
- module_target = '''
-using UnrealBuildTool;
-using System.Collections.Generic;
-
-public class {0}Target : TargetRules
-{{
- public {0}Target( TargetInfo Target) : base(Target)
- {{
- Type = TargetType.Game;
- ExtraModuleNames.AddRange( new string[] {{ "{0}" }} );
- }}
-}}
-'''.format(project_name)
-
- editor_module_target = '''
-using UnrealBuildTool;
-using System.Collections.Generic;
-
-public class {0}EditorTarget : TargetRules
-{{
- public {0}EditorTarget( TargetInfo Target) : base(Target)
- {{
- Type = TargetType.Editor;
-
- ExtraModuleNames.AddRange( new string[] {{ "{0}" }} );
- }}
-}}
-'''.format(project_name)
-
- module_build = '''
-using UnrealBuildTool;
-public class {0} : ModuleRules
-{{
- public {0}(ReadOnlyTargetRules Target) : base(Target)
- {{
- PCHUsage = PCHUsageMode.UseExplicitOrSharedPCHs;
- PublicDependencyModuleNames.AddRange(new string[] {{ "Core",
- "CoreUObject", "Engine", "InputCore" }});
- PrivateDependencyModuleNames.AddRange(new string[] {{ }});
- }}
-}}
-'''.format(project_name)
-
- module_cpp = '''
-#include "{0}.h"
-#include "Modules/ModuleManager.h"
-
-IMPLEMENT_PRIMARY_GAME_MODULE( FDefaultGameModuleImpl, {0}, "{0}" );
-'''.format(project_name)
-
- module_header = '''
-#pragma once
-#include "CoreMinimal.h"
-'''
-
- game_mode_cpp = '''
-#include "{0}GameModeBase.h"
-'''.format(project_name)
-
- game_mode_h = '''
-#pragma once
-
-#include "CoreMinimal.h"
-#include "GameFramework/GameModeBase.h"
-#include "{0}GameModeBase.generated.h"
-
-UCLASS()
-class {1}_API A{0}GameModeBase : public AGameModeBase
-{{
- GENERATED_BODY()
-}};
-'''.format(project_name, project_name.upper())
-
- with open(targets_dir / f"{project_name}.Target.cs", mode="w") as f:
- f.write(module_target)
-
- with open(targets_dir / f"{project_name}Editor.Target.cs", mode="w") as f:
- f.write(editor_module_target)
-
- with open(sources_dir / f"{project_name}.Build.cs", mode="w") as f:
- f.write(module_build)
-
- with open(sources_dir / f"{project_name}.cpp", mode="w") as f:
- f.write(module_cpp)
-
- with open(sources_dir / f"{project_name}.h", mode="w") as f:
- f.write(module_header)
-
- with open(sources_dir / f"{project_name}GameModeBase.cpp", mode="w") as f:
- f.write(game_mode_cpp)
-
- with open(sources_dir / f"{project_name}GameModeBase.h", mode="w") as f:
- f.write(game_mode_h)
-
- u_build_tool = Path(
- engine_path / "Engine/Binaries/DotNET/UnrealBuildTool.exe")
- u_header_tool = None
-
- arch = "Win64"
- if platform.system().lower() == "windows":
- arch = "Win64"
- u_header_tool = Path(
- engine_path / "Engine/Binaries/Win64/UnrealHeaderTool.exe")
- elif platform.system().lower() == "linux":
- arch = "Linux"
- u_header_tool = Path(
- engine_path / "Engine/Binaries/Linux/UnrealHeaderTool")
- elif platform.system().lower() == "darwin":
- # we need to test this out
- arch = "Mac"
- u_header_tool = Path(
- engine_path / "Engine/Binaries/Mac/UnrealHeaderTool")
-
- if not u_header_tool:
- raise NotImplementedError("Unsupported platform")
-
- command1 = [u_build_tool.as_posix(), "-projectfiles",
- f"-project={project_file}", "-progress"]
-
- subprocess.run(command1)
-
- command2 = [u_build_tool.as_posix(),
- f"-ModuleWithSuffix={project_name},3555", arch,
- "Development", "-TargetType=Editor",
- f'-Project={project_file}',
- f'{project_file}',
- "-IgnoreJunk"]
-
- subprocess.run(command2)
-
- """
- uhtmanifest = os.path.join(os.path.dirname(project_file),
- f"{project_name}.uhtmanifest")
-
- command3 = [u_header_tool, f'"{project_file}"', f'"{uhtmanifest}"',
- "-Unattended", "-WarningsAsErrors", "-installed"]
-
- subprocess.run(command3)
- """
diff --git a/openpype/hosts/unreal/api/pipeline.py b/openpype/hosts/unreal/api/pipeline.py
new file mode 100644
index 00000000000..d396b64072c
--- /dev/null
+++ b/openpype/hosts/unreal/api/pipeline.py
@@ -0,0 +1,443 @@
+# -*- coding: utf-8 -*-
+import os
+import logging
+from typing import List
+
+import pyblish.api
+
+from openpype.pipeline import (
+ register_loader_plugin_path,
+ register_creator_plugin_path,
+ deregister_loader_plugin_path,
+ deregister_creator_plugin_path,
+ AVALON_CONTAINER_ID,
+)
+from openpype.tools.utils import host_tools
+import openpype.hosts.unreal
+from openpype.host import HostBase, ILoadHost
+
+import unreal # noqa
+
+
+logger = logging.getLogger("openpype.hosts.unreal")
+OPENPYPE_CONTAINERS = "OpenPypeContainers"
+
+HOST_DIR = os.path.dirname(os.path.abspath(openpype.hosts.unreal.__file__))
+PLUGINS_DIR = os.path.join(HOST_DIR, "plugins")
+PUBLISH_PATH = os.path.join(PLUGINS_DIR, "publish")
+LOAD_PATH = os.path.join(PLUGINS_DIR, "load")
+CREATE_PATH = os.path.join(PLUGINS_DIR, "create")
+INVENTORY_PATH = os.path.join(PLUGINS_DIR, "inventory")
+
+
+class UnrealHost(HostBase, ILoadHost):
+ """Unreal host implementation.
+
+ For some time this class will re-use functions from module based
+ implementation for backwards compatibility of older unreal projects.
+ """
+
+ name = "unreal"
+
+ def install(self):
+ install()
+
+ def get_containers(self):
+ return ls()
+
+ def show_tools_popup(self):
+ """Show tools popup with actions leading to show other tools."""
+
+ show_tools_popup()
+
+ def show_tools_dialog(self):
+ """Show tools dialog with actions leading to show other tools."""
+
+ show_tools_dialog()
+
+
+def install():
+ """Install Unreal configuration for OpenPype."""
+ print("-=" * 40)
+ logo = '''.
+.
+ ____________
+ / \\ __ \\
+ \\ \\ \\/_\\ \\
+ \\ \\ _____/ ______
+ \\ \\ \\___// \\ \\
+ \\ \\____\\ \\ \\_____\\
+ \\/_____/ \\/______/ PYPE Club .
+.
+'''
+ print(logo)
+ print("installing OpenPype for Unreal ...")
+ print("-=" * 40)
+ logger.info("installing OpenPype for Unreal")
+ pyblish.api.register_host("unreal")
+ pyblish.api.register_plugin_path(str(PUBLISH_PATH))
+ register_loader_plugin_path(str(LOAD_PATH))
+ register_creator_plugin_path(str(CREATE_PATH))
+ _register_callbacks()
+ _register_events()
+
+
+def uninstall():
+ """Uninstall Unreal configuration for Avalon."""
+ pyblish.api.deregister_plugin_path(str(PUBLISH_PATH))
+ deregister_loader_plugin_path(str(LOAD_PATH))
+ deregister_creator_plugin_path(str(CREATE_PATH))
+
+
+def _register_callbacks():
+ """
+ TODO: Implement callbacks if supported by UE4
+ """
+ pass
+
+
+def _register_events():
+ """
+ TODO: Implement callbacks if supported by UE4
+ """
+ pass
+
+
+def ls():
+ """List all containers.
+
+ List all found in *Content Manager* of Unreal and return
+ metadata from them. Adding `objectName` to set.
+
+ """
+ ar = unreal.AssetRegistryHelpers.get_asset_registry()
+ openpype_containers = ar.get_assets_by_class("AssetContainer", True)
+
+ # get_asset_by_class returns AssetData. To get all metadata we need to
+ # load asset. get_tag_values() work only on metadata registered in
+ # Asset Registry Project settings (and there is no way to set it with
+ # python short of editing ini configuration file).
+ for asset_data in openpype_containers:
+ asset = asset_data.get_asset()
+ data = unreal.EditorAssetLibrary.get_metadata_tag_values(asset)
+ data["objectName"] = asset_data.asset_name
+ data = cast_map_to_str_dict(data)
+
+ yield data
+
+
+def parse_container(container):
+ """To get data from container, AssetContainer must be loaded.
+
+ Args:
+ container(str): path to container
+
+ Returns:
+ dict: metadata stored on container
+ """
+ asset = unreal.EditorAssetLibrary.load_asset(container)
+ data = unreal.EditorAssetLibrary.get_metadata_tag_values(asset)
+ data["objectName"] = asset.get_name()
+ data = cast_map_to_str_dict(data)
+
+ return data
+
+
+def publish():
+ """Shorthand to publish from within host."""
+ import pyblish.util
+
+ return pyblish.util.publish()
+
+
+def containerise(name, namespace, nodes, context, loader=None, suffix="_CON"):
+ """Bundles *nodes* (assets) into a *container* and add metadata to it.
+
+ Unreal doesn't support *groups* of assets that you can add metadata to.
+ But it does support folders that helps to organize asset. Unfortunately
+ those folders are just that - you cannot add any additional information
+ to them. OpenPype Integration Plugin is providing way out - Implementing
+ `AssetContainer` Blueprint class. This class when added to folder can
+ handle metadata on it using standard
+ :func:`unreal.EditorAssetLibrary.set_metadata_tag()` and
+ :func:`unreal.EditorAssetLibrary.get_metadata_tag_values()`. It also
+ stores and monitor all changes in assets in path where it resides. List of
+ those assets is available as `assets` property.
+
+ This is list of strings starting with asset type and ending with its path:
+ `Material /Game/OpenPype/Test/TestMaterial.TestMaterial`
+
+ """
+ # 1 - create directory for container
+ root = "/Game"
+ container_name = "{}{}".format(name, suffix)
+ new_name = move_assets_to_path(root, container_name, nodes)
+
+ # 2 - create Asset Container there
+ path = "{}/{}".format(root, new_name)
+ create_container(container=container_name, path=path)
+
+ namespace = path
+
+ data = {
+ "schema": "openpype:container-2.0",
+ "id": AVALON_CONTAINER_ID,
+ "name": new_name,
+ "namespace": namespace,
+ "loader": str(loader),
+ "representation": context["representation"]["_id"],
+ }
+ # 3 - imprint data
+ imprint("{}/{}".format(path, container_name), data)
+ return path
+
+
+def instantiate(root, name, data, assets=None, suffix="_INS"):
+ """Bundles *nodes* into *container*.
+
+ Marking it with metadata as publishable instance. If assets are provided,
+ they are moved to new path where `OpenPypePublishInstance` class asset is
+ created and imprinted with metadata.
+
+ This can then be collected for publishing by Pyblish for example.
+
+ Args:
+ root (str): root path where to create instance container
+ name (str): name of the container
+ data (dict): data to imprint on container
+ assets (list of str): list of asset paths to include in publish
+ instance
+ suffix (str): suffix string to append to instance name
+
+ """
+ container_name = "{}{}".format(name, suffix)
+
+ # if we specify assets, create new folder and move them there. If not,
+ # just create empty folder
+ if assets:
+ new_name = move_assets_to_path(root, container_name, assets)
+ else:
+ new_name = create_folder(root, name)
+
+ path = "{}/{}".format(root, new_name)
+ create_publish_instance(instance=container_name, path=path)
+
+ imprint("{}/{}".format(path, container_name), data)
+
+
+def imprint(node, data):
+ loaded_asset = unreal.EditorAssetLibrary.load_asset(node)
+ for key, value in data.items():
+ # Support values evaluated at imprint
+ if callable(value):
+ value = value()
+ # Unreal doesn't support NoneType in metadata values
+ if value is None:
+ value = ""
+ unreal.EditorAssetLibrary.set_metadata_tag(
+ loaded_asset, key, str(value)
+ )
+
+ with unreal.ScopedEditorTransaction("OpenPype containerising"):
+ unreal.EditorAssetLibrary.save_asset(node)
+
+
+def show_tools_popup():
+ """Show popup with tools.
+
+ Popup will disappear on click or loosing focus.
+ """
+ from openpype.hosts.unreal.api import tools_ui
+
+ tools_ui.show_tools_popup()
+
+
+def show_tools_dialog():
+ """Show dialog with tools.
+
+ Dialog will stay visible.
+ """
+ from openpype.hosts.unreal.api import tools_ui
+
+ tools_ui.show_tools_dialog()
+
+
+def show_creator():
+ host_tools.show_creator()
+
+
+def show_loader():
+ host_tools.show_loader(use_context=True)
+
+
+def show_publisher():
+ host_tools.show_publish()
+
+
+def show_manager():
+ host_tools.show_scene_inventory()
+
+
+def show_experimental_tools():
+ host_tools.show_experimental_tools_dialog()
+
+
+def create_folder(root: str, name: str) -> str:
+ """Create new folder.
+
+ If folder exists, append number at the end and try again, incrementing
+ if needed.
+
+ Args:
+ root (str): path root
+ name (str): folder name
+
+ Returns:
+ str: folder name
+
+ Example:
+ >>> create_folder("/Game/Foo")
+ /Game/Foo
+ >>> create_folder("/Game/Foo")
+ /Game/Foo1
+
+ """
+ eal = unreal.EditorAssetLibrary
+ index = 1
+ while True:
+ if eal.does_directory_exist("{}/{}".format(root, name)):
+ name = "{}{}".format(name, index)
+ index += 1
+ else:
+ eal.make_directory("{}/{}".format(root, name))
+ break
+
+ return name
+
+
+def move_assets_to_path(root: str, name: str, assets: List[str]) -> str:
+ """Moving (renaming) list of asset paths to new destination.
+
+ Args:
+ root (str): root of the path (eg. `/Game`)
+ name (str): name of destination directory (eg. `Foo` )
+ assets (list of str): list of asset paths
+
+ Returns:
+ str: folder name
+
+ Example:
+ This will get paths of all assets under `/Game/Test` and move them
+ to `/Game/NewTest`. If `/Game/NewTest` already exists, then resulting
+ path will be `/Game/NewTest1`
+
+ >>> assets = unreal.EditorAssetLibrary.list_assets("/Game/Test")
+ >>> move_assets_to_path("/Game", "NewTest", assets)
+ NewTest
+
+ """
+ eal = unreal.EditorAssetLibrary
+ name = create_folder(root, name)
+
+ unreal.log(assets)
+ for asset in assets:
+ loaded = eal.load_asset(asset)
+ eal.rename_asset(
+ asset, "{}/{}/{}".format(root, name, loaded.get_name())
+ )
+
+ return name
+
+
+def create_container(container: str, path: str) -> unreal.Object:
+ """Helper function to create Asset Container class on given path.
+
+ This Asset Class helps to mark given path as Container
+ and enable asset version control on it.
+
+ Args:
+ container (str): Asset Container name
+ path (str): Path where to create Asset Container. This path should
+ point into container folder
+
+ Returns:
+ :class:`unreal.Object`: instance of created asset
+
+ Example:
+
+ create_container(
+ "/Game/modelingFooCharacter_CON",
+ "modelingFooCharacter_CON"
+ )
+
+ """
+ factory = unreal.AssetContainerFactory()
+ tools = unreal.AssetToolsHelpers().get_asset_tools()
+
+ asset = tools.create_asset(container, path, None, factory)
+ return asset
+
+
+def create_publish_instance(instance: str, path: str) -> unreal.Object:
+ """Helper function to create OpenPype Publish Instance on given path.
+
+ This behaves similarly as :func:`create_openpype_container`.
+
+ Args:
+ path (str): Path where to create Publish Instance.
+ This path should point into container folder
+ instance (str): Publish Instance name
+
+ Returns:
+ :class:`unreal.Object`: instance of created asset
+
+ Example:
+
+ create_publish_instance(
+ "/Game/modelingFooCharacter_INST",
+ "modelingFooCharacter_INST"
+ )
+
+ """
+ factory = unreal.OpenPypePublishInstanceFactory()
+ tools = unreal.AssetToolsHelpers().get_asset_tools()
+ asset = tools.create_asset(instance, path, None, factory)
+ return asset
+
+
+def cast_map_to_str_dict(umap) -> dict:
+ """Cast Unreal Map to dict.
+
+ Helper function to cast Unreal Map object to plain old python
+ dict. This will also cast values and keys to str. Useful for
+ metadata dicts.
+
+ Args:
+ umap: Unreal Map object
+
+ Returns:
+ dict
+
+ """
+ return {str(key): str(value) for (key, value) in umap.items()}
+
+
+def get_subsequences(sequence: unreal.LevelSequence):
+ """Get list of subsequences from sequence.
+
+ Args:
+ sequence (unreal.LevelSequence): Sequence
+
+ Returns:
+ list(unreal.LevelSequence): List of subsequences
+
+ """
+ tracks = sequence.get_master_tracks()
+ subscene_track = None
+ for t in tracks:
+ if t.get_class() == unreal.MovieSceneSubTrack.static_class():
+ subscene_track = t
+ break
+ if subscene_track is not None and subscene_track.get_sections():
+ return subscene_track.get_sections()
+ return []
diff --git a/openpype/hosts/unreal/api/plugin.py b/openpype/hosts/unreal/api/plugin.py
index a69b12d6f51..d8d2f2420de 100644
--- a/openpype/hosts/unreal/api/plugin.py
+++ b/openpype/hosts/unreal/api/plugin.py
@@ -1,12 +1,18 @@
-from avalon import api
-import openpype.api
+# -*- coding: utf-8 -*-
+from abc import ABC
+from openpype.pipeline import (
+ LegacyCreator,
+ LoaderPlugin,
+)
-class Creator(openpype.api.Creator):
+
+class Creator(LegacyCreator):
"""This serves as skeleton for future OpenPype specific functionality"""
- pass
+ defaults = ['Main']
+ maintain_selection = False
-class Loader(api.Loader):
+class Loader(LoaderPlugin, ABC):
"""This serves as skeleton for future OpenPype specific functionality"""
pass
diff --git a/openpype/hosts/unreal/api/rendering.py b/openpype/hosts/unreal/api/rendering.py
new file mode 100644
index 00000000000..29e4747f6ec
--- /dev/null
+++ b/openpype/hosts/unreal/api/rendering.py
@@ -0,0 +1,137 @@
+import os
+
+import unreal
+
+from openpype.pipeline import Anatomy
+from openpype.hosts.unreal.api import pipeline
+
+
+queue = None
+executor = None
+
+
+def _queue_finish_callback(exec, success):
+ unreal.log("Render completed. Success: " + str(success))
+
+ # Delete our reference so we don't keep it alive.
+ global executor
+ global queue
+ del executor
+ del queue
+
+
+def _job_finish_callback(job, success):
+ # You can make any edits you want to the editor world here, and the world
+ # will be duplicated when the next render happens. Make sure you undo your
+ # edits in OnQueueFinishedCallback if you don't want to leak state changes
+ # into the editor world.
+ unreal.log("Individual job completed.")
+
+
+def start_rendering():
+ """
+ Start the rendering process.
+ """
+ print("Starting rendering...")
+
+ # Get selected sequences
+ assets = unreal.EditorUtilityLibrary.get_selected_assets()
+
+ # instances = pipeline.ls_inst()
+ instances = [
+ a for a in assets
+ if a.get_class().get_name() == "OpenPypePublishInstance"]
+
+ inst_data = []
+
+ for i in instances:
+ data = pipeline.parse_container(i.get_path_name())
+ if data["family"] == "render":
+ inst_data.append(data)
+
+ try:
+ project = os.environ.get("AVALON_PROJECT")
+ anatomy = Anatomy(project)
+ root = anatomy.roots['renders']
+ except Exception:
+ raise Exception("Could not find render root in anatomy settings.")
+
+ render_dir = f"{root}/{project}"
+
+ # subsystem = unreal.get_editor_subsystem(
+ # unreal.MoviePipelineQueueSubsystem)
+ # queue = subsystem.get_queue()
+ global queue
+ queue = unreal.MoviePipelineQueue()
+
+ ar = unreal.AssetRegistryHelpers.get_asset_registry()
+
+ for i in inst_data:
+ sequence = ar.get_asset_by_object_path(i["sequence"]).get_asset()
+
+ sequences = [{
+ "sequence": sequence,
+ "output": f"{i['output']}",
+ "frame_range": (
+ int(float(i["frameStart"])),
+ int(float(i["frameEnd"])) + 1)
+ }]
+ render_list = []
+
+ # Get all the sequences to render. If there are subsequences,
+ # add them and their frame ranges to the render list. We also
+ # use the names for the output paths.
+ for s in sequences:
+ subscenes = pipeline.get_subsequences(s.get('sequence'))
+
+ if subscenes:
+ for ss in subscenes:
+ sequences.append({
+ "sequence": ss.get_sequence(),
+ "output": (f"{s.get('output')}/"
+ f"{ss.get_sequence().get_name()}"),
+ "frame_range": (
+ ss.get_start_frame(), ss.get_end_frame())
+ })
+ else:
+ # Avoid rendering camera sequences
+ if "_camera" not in s.get('sequence').get_name():
+ render_list.append(s)
+
+ # Create the rendering jobs and add them to the queue.
+ for r in render_list:
+ job = queue.allocate_new_job(unreal.MoviePipelineExecutorJob)
+ job.sequence = unreal.SoftObjectPath(i["master_sequence"])
+ job.map = unreal.SoftObjectPath(i["master_level"])
+ job.author = "OpenPype"
+
+ # User data could be used to pass data to the job, that can be
+ # read in the job's OnJobFinished callback. We could,
+ # for instance, pass the AvalonPublishInstance's path to the job.
+ # job.user_data = ""
+
+ settings = job.get_configuration().find_or_add_setting_by_class(
+ unreal.MoviePipelineOutputSetting)
+ settings.output_resolution = unreal.IntPoint(1920, 1080)
+ settings.custom_start_frame = r.get("frame_range")[0]
+ settings.custom_end_frame = r.get("frame_range")[1]
+ settings.use_custom_playback_range = True
+ settings.file_name_format = "{sequence_name}.{frame_number}"
+ settings.output_directory.path = f"{render_dir}/{r.get('output')}"
+
+ renderPass = job.get_configuration().find_or_add_setting_by_class(
+ unreal.MoviePipelineDeferredPassBase)
+ renderPass.disable_multisample_effects = True
+
+ job.get_configuration().find_or_add_setting_by_class(
+ unreal.MoviePipelineImageSequenceOutput_PNG)
+
+ # If there are jobs in the queue, start the rendering process.
+ if queue.get_jobs():
+ global executor
+ executor = unreal.MoviePipelinePIEExecutor()
+ executor.on_executor_finished_delegate.add_callable_unique(
+ _queue_finish_callback)
+ executor.on_individual_job_finished_delegate.add_callable_unique(
+ _job_finish_callback) # Only available on PIE Executor
+ executor.execute(queue)
diff --git a/openpype/hosts/unreal/api/tools_ui.py b/openpype/hosts/unreal/api/tools_ui.py
index 93361c3574e..2500f8495f6 100644
--- a/openpype/hosts/unreal/api/tools_ui.py
+++ b/openpype/hosts/unreal/api/tools_ui.py
@@ -7,6 +7,7 @@
)
from openpype.tools.utils import host_tools
from openpype.tools.utils.lib import qt_app_context
+from openpype.hosts.unreal.api import rendering
class ToolsBtnsWidget(QtWidgets.QWidget):
@@ -20,6 +21,7 @@ def __init__(self, parent=None):
load_btn = QtWidgets.QPushButton("Load...", self)
publish_btn = QtWidgets.QPushButton("Publish...", self)
manage_btn = QtWidgets.QPushButton("Manage...", self)
+ render_btn = QtWidgets.QPushButton("Render...", self)
experimental_tools_btn = QtWidgets.QPushButton(
"Experimental tools...", self
)
@@ -30,6 +32,7 @@ def __init__(self, parent=None):
layout.addWidget(load_btn, 0)
layout.addWidget(publish_btn, 0)
layout.addWidget(manage_btn, 0)
+ layout.addWidget(render_btn, 0)
layout.addWidget(experimental_tools_btn, 0)
layout.addStretch(1)
@@ -37,6 +40,7 @@ def __init__(self, parent=None):
load_btn.clicked.connect(self._on_load)
publish_btn.clicked.connect(self._on_publish)
manage_btn.clicked.connect(self._on_manage)
+ render_btn.clicked.connect(self._on_render)
experimental_tools_btn.clicked.connect(self._on_experimental)
def _on_create(self):
@@ -51,6 +55,9 @@ def _on_publish(self):
def _on_manage(self):
self.tool_required.emit("sceneinventory")
+ def _on_render(self):
+ rendering.start_rendering()
+
def _on_experimental(self):
self.tool_required.emit("experimental_tools")
diff --git a/openpype/hosts/unreal/hooks/pre_workfile_preparation.py b/openpype/hosts/unreal/hooks/pre_workfile_preparation.py
index 880dba5cfbf..50b34bd573d 100644
--- a/openpype/hosts/unreal/hooks/pre_workfile_preparation.py
+++ b/openpype/hosts/unreal/hooks/pre_workfile_preparation.py
@@ -1,16 +1,16 @@
# -*- coding: utf-8 -*-
"""Hook to launch Unreal and prepare projects."""
import os
+import copy
from pathlib import Path
from openpype.lib import (
PreLaunchHook,
ApplicationLaunchFailed,
ApplicationNotFound,
- get_workdir_data,
get_workfile_template_key
)
-from openpype.hosts.unreal.api import lib as unreal_lib
+import openpype.hosts.unreal.lib as unreal_lib
class UnrealPrelaunchHook(PreLaunchHook):
@@ -25,7 +25,7 @@ class UnrealPrelaunchHook(PreLaunchHook):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
- self.signature = "( {} )".format(self.__class__.__name__)
+ self.signature = f"( {self.__class__.__name__} )"
def _get_work_filename(self):
# Use last workfile if was found
@@ -35,18 +35,13 @@ def _get_work_filename(self):
return last_workfile.name
# Prepare data for fill data and for getting workfile template key
- task_name = self.data["task_name"]
anatomy = self.data["anatomy"]
- asset_doc = self.data["asset_doc"]
project_doc = self.data["project_doc"]
- asset_tasks = asset_doc.get("data", {}).get("tasks") or {}
- task_info = asset_tasks.get(task_name) or {}
- task_type = task_info.get("type")
+ # Use already prepared workdir data
+ workdir_data = copy.deepcopy(self.data["workdir_data"])
+ task_type = workdir_data.get("task", {}).get("type")
- workdir_data = get_workdir_data(
- project_doc, asset_doc, task_name, self.host_name
- )
# QUESTION raise exception if version is part of filename template?
workdir_data["version"] = 1
workdir_data["ext"] = "uproject"
@@ -71,7 +66,7 @@ def execute(self):
if int(engine_version.split(".")[0]) < 4 and \
int(engine_version.split(".")[1]) < 26:
raise ApplicationLaunchFailed((
- f"{self.signature} Old unsupported version of UE4 "
+ f"{self.signature} Old unsupported version of UE "
f"detected - {engine_version}"))
except ValueError:
# there can be string in minor version and in that case
@@ -99,18 +94,19 @@ def execute(self):
f"character ({unreal_project_name}). Appending 'P'"
))
unreal_project_name = f"P{unreal_project_name}"
+ unreal_project_filename = f'{unreal_project_name}.uproject'
project_path = Path(os.path.join(workdir, unreal_project_name))
self.log.info((
- f"{self.signature} requested UE4 version: "
+ f"{self.signature} requested UE version: "
f"[ {engine_version} ]"
))
detected = unreal_lib.get_engine_versions(self.launch_context.env)
detected_str = ', '.join(detected.keys()) or 'none'
self.log.info((
- f"{self.signature} detected UE4 versions: "
+ f"{self.signature} detected UE versions: "
f"[ {detected_str} ]"
))
if not detected:
@@ -123,10 +119,10 @@ def execute(self):
f"detected [ {engine_version} ]"
))
- ue4_path = unreal_lib.get_editor_executable_path(
- Path(detected[engine_version]))
+ ue_path = unreal_lib.get_editor_executable_path(
+ Path(detected[engine_version]), engine_version)
- self.launch_context.launch_args = [ue4_path.as_posix()]
+ self.launch_context.launch_args = [ue_path.as_posix()]
project_path.mkdir(parents=True, exist_ok=True)
project_file = project_path / unreal_project_filename
@@ -136,9 +132,14 @@ def execute(self):
f"{self.signature} creating unreal "
f"project [ {unreal_project_name} ]"
))
- # Set "AVALON_UNREAL_PLUGIN" to current process environment for
+ # Set "OPENPYPE_UNREAL_PLUGIN" to current process environment for
# execution of `create_unreal_project`
- env_key = "AVALON_UNREAL_PLUGIN"
+ if self.launch_context.env.get("OPENPYPE_UNREAL_PLUGIN"):
+ self.log.info((
+ f"{self.signature} using OpenPype plugin from "
+ f"{self.launch_context.env.get('OPENPYPE_UNREAL_PLUGIN')}"
+ ))
+ env_key = "OPENPYPE_UNREAL_PLUGIN"
if self.launch_context.env.get(env_key):
os.environ[env_key] = self.launch_context.env[env_key]
diff --git a/openpype/hosts/unreal/integration/UE_4.7/.gitignore b/openpype/hosts/unreal/integration/UE_4.7/.gitignore
new file mode 100644
index 00000000000..b32a6f55e5d
--- /dev/null
+++ b/openpype/hosts/unreal/integration/UE_4.7/.gitignore
@@ -0,0 +1,35 @@
+# Prerequisites
+*.d
+
+# Compiled Object files
+*.slo
+*.lo
+*.o
+*.obj
+
+# Precompiled Headers
+*.gch
+*.pch
+
+# Compiled Dynamic libraries
+*.so
+*.dylib
+*.dll
+
+# Fortran module files
+*.mod
+*.smod
+
+# Compiled Static libraries
+*.lai
+*.la
+*.a
+*.lib
+
+# Executables
+*.exe
+*.out
+*.app
+
+/Binaries
+/Intermediate
diff --git a/openpype/hosts/unreal/integration/UE_4.7/Content/Python/init_unreal.py b/openpype/hosts/unreal/integration/UE_4.7/Content/Python/init_unreal.py
new file mode 100644
index 00000000000..b85f9706992
--- /dev/null
+++ b/openpype/hosts/unreal/integration/UE_4.7/Content/Python/init_unreal.py
@@ -0,0 +1,30 @@
+import unreal
+
+openpype_detected = True
+try:
+ from openpype.pipeline import install_host
+ from openpype.hosts.unreal.api import UnrealHost
+
+ openpype_host = UnrealHost()
+except ImportError as exc:
+ openpype_host = None
+ openpype_detected = False
+ unreal.log_error("OpenPype: cannot load OpenPype [ {} ]".format(exc))
+
+if openpype_detected:
+ install_host(openpype_host)
+
+
+@unreal.uclass()
+class OpenPypeIntegration(unreal.OpenPypePythonBridge):
+ @unreal.ufunction(override=True)
+ def RunInPython_Popup(self):
+ unreal.log_warning("OpenPype: showing tools popup")
+ if openpype_detected:
+ openpype_host.show_tools_popup()
+
+ @unreal.ufunction(override=True)
+ def RunInPython_Dialog(self):
+ unreal.log_warning("OpenPype: showing tools dialog")
+ if openpype_detected:
+ openpype_host.show_tools_dialog()
diff --git a/openpype/hosts/unreal/integration/UE_4.7/OpenPype.uplugin b/openpype/hosts/unreal/integration/UE_4.7/OpenPype.uplugin
new file mode 100644
index 00000000000..4c7a74403c2
--- /dev/null
+++ b/openpype/hosts/unreal/integration/UE_4.7/OpenPype.uplugin
@@ -0,0 +1,24 @@
+{
+ "FileVersion": 3,
+ "Version": 1,
+ "VersionName": "1.0",
+ "FriendlyName": "OpenPype",
+ "Description": "OpenPype Integration",
+ "Category": "OpenPype.Integration",
+ "CreatedBy": "Ondrej Samohel",
+ "CreatedByURL": "https://openpype.io",
+ "DocsURL": "https://openpype.io/docs/artist_hosts_unreal",
+ "MarketplaceURL": "",
+ "SupportURL": "https://pype.club/",
+ "CanContainContent": true,
+ "IsBetaVersion": true,
+ "IsExperimentalVersion": false,
+ "Installed": false,
+ "Modules": [
+ {
+ "Name": "OpenPype",
+ "Type": "Editor",
+ "LoadingPhase": "Default"
+ }
+ ]
+}
\ No newline at end of file
diff --git a/openpype/hosts/unreal/integration/UE_4.7/README.md b/openpype/hosts/unreal/integration/UE_4.7/README.md
new file mode 100644
index 00000000000..a08c1ada390
--- /dev/null
+++ b/openpype/hosts/unreal/integration/UE_4.7/README.md
@@ -0,0 +1,11 @@
+# OpenPype Unreal Integration plugin - UE 4.x
+
+This is plugin for Unreal Editor, creating menu for [OpenPype](https://github.com/getavalon) tools to run.
+
+## How does this work
+
+Plugin is creating basic menu items in **Window/OpenPype** section of Unreal Editor main menu and a button
+on the main toolbar with associated menu. Clicking on those menu items is calling callbacks that are
+declared in c++ but needs to be implemented during Unreal Editor
+startup in `Plugins/OpenPype/Content/Python/init_unreal.py` - this should be executed by Unreal Editor
+automatically.
diff --git a/openpype/hosts/unreal/integration/UE_4.7/Resources/openpype128.png b/openpype/hosts/unreal/integration/UE_4.7/Resources/openpype128.png
new file mode 100644
index 00000000000..abe8a807ef4
Binary files /dev/null and b/openpype/hosts/unreal/integration/UE_4.7/Resources/openpype128.png differ
diff --git a/openpype/hosts/unreal/integration/UE_4.7/Resources/openpype40.png b/openpype/hosts/unreal/integration/UE_4.7/Resources/openpype40.png
new file mode 100644
index 00000000000..f983e7a1f20
Binary files /dev/null and b/openpype/hosts/unreal/integration/UE_4.7/Resources/openpype40.png differ
diff --git a/openpype/hosts/unreal/integration/UE_4.7/Resources/openpype512.png b/openpype/hosts/unreal/integration/UE_4.7/Resources/openpype512.png
new file mode 100644
index 00000000000..97c4d4326bc
Binary files /dev/null and b/openpype/hosts/unreal/integration/UE_4.7/Resources/openpype512.png differ
diff --git a/openpype/hosts/unreal/integration/UE_4.7/Source/OpenPype/OpenPype.Build.cs b/openpype/hosts/unreal/integration/UE_4.7/Source/OpenPype/OpenPype.Build.cs
new file mode 100644
index 00000000000..c30835b63dc
--- /dev/null
+++ b/openpype/hosts/unreal/integration/UE_4.7/Source/OpenPype/OpenPype.Build.cs
@@ -0,0 +1,57 @@
+// Copyright 1998-2019 Epic Games, Inc. All Rights Reserved.
+
+using UnrealBuildTool;
+
+public class OpenPype : ModuleRules
+{
+ public OpenPype(ReadOnlyTargetRules Target) : base(Target)
+ {
+ PCHUsage = ModuleRules.PCHUsageMode.UseExplicitOrSharedPCHs;
+
+ PublicIncludePaths.AddRange(
+ new string[] {
+ // ... add public include paths required here ...
+ }
+ );
+
+
+ PrivateIncludePaths.AddRange(
+ new string[] {
+ // ... add other private include paths required here ...
+ }
+ );
+
+
+ PublicDependencyModuleNames.AddRange(
+ new string[]
+ {
+ "Core",
+ // ... add other public dependencies that you statically link with here ...
+ }
+ );
+
+
+ PrivateDependencyModuleNames.AddRange(
+ new string[]
+ {
+ "Projects",
+ "InputCore",
+ "UnrealEd",
+ "LevelEditor",
+ "CoreUObject",
+ "Engine",
+ "Slate",
+ "SlateCore",
+ // ... add private dependencies that you statically link with here ...
+ }
+ );
+
+
+ DynamicallyLoadedModuleNames.AddRange(
+ new string[]
+ {
+ // ... add any modules that your module loads dynamically here ...
+ }
+ );
+ }
+}
diff --git a/openpype/hosts/unreal/integration/UE_4.7/Source/OpenPype/Private/AssetContainer.cpp b/openpype/hosts/unreal/integration/UE_4.7/Source/OpenPype/Private/AssetContainer.cpp
new file mode 100644
index 00000000000..c766f87a8ef
--- /dev/null
+++ b/openpype/hosts/unreal/integration/UE_4.7/Source/OpenPype/Private/AssetContainer.cpp
@@ -0,0 +1,115 @@
+// Fill out your copyright notice in the Description page of Project Settings.
+
+#include "AssetContainer.h"
+#include "AssetRegistryModule.h"
+#include "Misc/PackageName.h"
+#include "Engine.h"
+#include "Containers/UnrealString.h"
+
+UAssetContainer::UAssetContainer(const FObjectInitializer& ObjectInitializer)
+: UAssetUserData(ObjectInitializer)
+{
+ FAssetRegistryModule& AssetRegistryModule = FModuleManager::LoadModuleChecked("AssetRegistry");
+ FString path = UAssetContainer::GetPathName();
+ UE_LOG(LogTemp, Warning, TEXT("UAssetContainer %s"), *path);
+ FARFilter Filter;
+ Filter.PackagePaths.Add(FName(*path));
+
+ AssetRegistryModule.Get().OnAssetAdded().AddUObject(this, &UAssetContainer::OnAssetAdded);
+ AssetRegistryModule.Get().OnAssetRemoved().AddUObject(this, &UAssetContainer::OnAssetRemoved);
+ AssetRegistryModule.Get().OnAssetRenamed().AddUObject(this, &UAssetContainer::OnAssetRenamed);
+}
+
+void UAssetContainer::OnAssetAdded(const FAssetData& AssetData)
+{
+ TArray split;
+
+ // get directory of current container
+ FString selfFullPath = UAssetContainer::GetPathName();
+ FString selfDir = FPackageName::GetLongPackagePath(*selfFullPath);
+
+ // get asset path and class
+ FString assetPath = AssetData.GetFullName();
+ FString assetFName = AssetData.AssetClass.ToString();
+
+ // split path
+ assetPath.ParseIntoArray(split, TEXT(" "), true);
+
+ FString assetDir = FPackageName::GetLongPackagePath(*split[1]);
+
+ // take interest only in paths starting with path of current container
+ if (assetDir.StartsWith(*selfDir))
+ {
+ // exclude self
+ if (assetFName != "AssetContainer")
+ {
+ assets.Add(assetPath);
+ assetsData.Add(AssetData);
+ UE_LOG(LogTemp, Log, TEXT("%s: asset added to %s"), *selfFullPath, *selfDir);
+ }
+ }
+}
+
+void UAssetContainer::OnAssetRemoved(const FAssetData& AssetData)
+{
+ TArray split;
+
+ // get directory of current container
+ FString selfFullPath = UAssetContainer::GetPathName();
+ FString selfDir = FPackageName::GetLongPackagePath(*selfFullPath);
+
+ // get asset path and class
+ FString assetPath = AssetData.GetFullName();
+ FString assetFName = AssetData.AssetClass.ToString();
+
+ // split path
+ assetPath.ParseIntoArray(split, TEXT(" "), true);
+
+ FString assetDir = FPackageName::GetLongPackagePath(*split[1]);
+
+ // take interest only in paths starting with path of current container
+ FString path = UAssetContainer::GetPathName();
+ FString lpp = FPackageName::GetLongPackagePath(*path);
+
+ if (assetDir.StartsWith(*selfDir))
+ {
+ // exclude self
+ if (assetFName != "AssetContainer")
+ {
+ // UE_LOG(LogTemp, Warning, TEXT("%s: asset removed"), *lpp);
+ assets.Remove(assetPath);
+ assetsData.Remove(AssetData);
+ }
+ }
+}
+
+void UAssetContainer::OnAssetRenamed(const FAssetData& AssetData, const FString& str)
+{
+ TArray split;
+
+ // get directory of current container
+ FString selfFullPath = UAssetContainer::GetPathName();
+ FString selfDir = FPackageName::GetLongPackagePath(*selfFullPath);
+
+ // get asset path and class
+ FString assetPath = AssetData.GetFullName();
+ FString assetFName = AssetData.AssetClass.ToString();
+
+ // split path
+ assetPath.ParseIntoArray(split, TEXT(" "), true);
+
+ FString assetDir = FPackageName::GetLongPackagePath(*split[1]);
+ if (assetDir.StartsWith(*selfDir))
+ {
+ // exclude self
+ if (assetFName != "AssetContainer")
+ {
+
+ assets.Remove(str);
+ assets.Add(assetPath);
+ assetsData.Remove(AssetData);
+ // UE_LOG(LogTemp, Warning, TEXT("%s: asset renamed %s"), *lpp, *str);
+ }
+ }
+}
+
diff --git a/openpype/hosts/unreal/integration/UE_4.7/Source/OpenPype/Private/AssetContainerFactory.cpp b/openpype/hosts/unreal/integration/UE_4.7/Source/OpenPype/Private/AssetContainerFactory.cpp
new file mode 100644
index 00000000000..b943150bdd0
--- /dev/null
+++ b/openpype/hosts/unreal/integration/UE_4.7/Source/OpenPype/Private/AssetContainerFactory.cpp
@@ -0,0 +1,20 @@
+#include "AssetContainerFactory.h"
+#include "AssetContainer.h"
+
+UAssetContainerFactory::UAssetContainerFactory(const FObjectInitializer& ObjectInitializer)
+ : UFactory(ObjectInitializer)
+{
+ SupportedClass = UAssetContainer::StaticClass();
+ bCreateNew = false;
+ bEditorImport = true;
+}
+
+UObject* UAssetContainerFactory::FactoryCreateNew(UClass* Class, UObject* InParent, FName Name, EObjectFlags Flags, UObject* Context, FFeedbackContext* Warn)
+{
+ UAssetContainer* AssetContainer = NewObject(InParent, Class, Name, Flags);
+ return AssetContainer;
+}
+
+bool UAssetContainerFactory::ShouldShowInNewMenu() const {
+ return false;
+}
diff --git a/openpype/hosts/unreal/integration/UE_4.7/Source/OpenPype/Private/OpenPype.cpp b/openpype/hosts/unreal/integration/UE_4.7/Source/OpenPype/Private/OpenPype.cpp
new file mode 100644
index 00000000000..15c46b38628
--- /dev/null
+++ b/openpype/hosts/unreal/integration/UE_4.7/Source/OpenPype/Private/OpenPype.cpp
@@ -0,0 +1,103 @@
+#include "OpenPype.h"
+#include "LevelEditor.h"
+#include "OpenPypePythonBridge.h"
+#include "OpenPypeStyle.h"
+
+
+static const FName OpenPypeTabName("OpenPype");
+
+#define LOCTEXT_NAMESPACE "FOpenPypeModule"
+
+// This function is triggered when the plugin is staring up
+void FOpenPypeModule::StartupModule()
+{
+
+ FOpenPypeStyle::Initialize();
+ FOpenPypeStyle::SetIcon("Logo", "openpype40");
+
+ // Create the Extender that will add content to the menu
+ FLevelEditorModule& LevelEditorModule = FModuleManager::LoadModuleChecked("LevelEditor");
+
+ TSharedPtr MenuExtender = MakeShareable(new FExtender());
+ TSharedPtr ToolbarExtender = MakeShareable(new FExtender());
+
+ MenuExtender->AddMenuExtension(
+ "LevelEditor",
+ EExtensionHook::After,
+ NULL,
+ FMenuExtensionDelegate::CreateRaw(this, &FOpenPypeModule::AddMenuEntry)
+ );
+ ToolbarExtender->AddToolBarExtension(
+ "Settings",
+ EExtensionHook::After,
+ NULL,
+ FToolBarExtensionDelegate::CreateRaw(this, &FOpenPypeModule::AddToobarEntry));
+
+
+ LevelEditorModule.GetMenuExtensibilityManager()->AddExtender(MenuExtender);
+ LevelEditorModule.GetToolBarExtensibilityManager()->AddExtender(ToolbarExtender);
+
+}
+
+void FOpenPypeModule::ShutdownModule()
+{
+ FOpenPypeStyle::Shutdown();
+}
+
+
+void FOpenPypeModule::AddMenuEntry(FMenuBuilder& MenuBuilder)
+{
+ // Create Section
+ MenuBuilder.BeginSection("OpenPype", TAttribute(FText::FromString("OpenPype")));
+ {
+ // Create a Submenu inside of the Section
+ MenuBuilder.AddMenuEntry(
+ FText::FromString("Tools..."),
+ FText::FromString("Pipeline tools"),
+ FSlateIcon(FOpenPypeStyle::GetStyleSetName(), "OpenPype.Logo"),
+ FUIAction(FExecuteAction::CreateRaw(this, &FOpenPypeModule::MenuPopup))
+ );
+
+ MenuBuilder.AddMenuEntry(
+ FText::FromString("Tools dialog..."),
+ FText::FromString("Pipeline tools dialog"),
+ FSlateIcon(FOpenPypeStyle::GetStyleSetName(), "OpenPype.Logo"),
+ FUIAction(FExecuteAction::CreateRaw(this, &FOpenPypeModule::MenuDialog))
+ );
+
+ }
+ MenuBuilder.EndSection();
+}
+
+void FOpenPypeModule::AddToobarEntry(FToolBarBuilder& ToolbarBuilder)
+{
+ ToolbarBuilder.BeginSection(TEXT("OpenPype"));
+ {
+ ToolbarBuilder.AddToolBarButton(
+ FUIAction(
+ FExecuteAction::CreateRaw(this, &FOpenPypeModule::MenuPopup),
+ NULL,
+ FIsActionChecked()
+
+ ),
+ NAME_None,
+ LOCTEXT("OpenPype_label", "OpenPype"),
+ LOCTEXT("OpenPype_tooltip", "OpenPype Tools"),
+ FSlateIcon(FOpenPypeStyle::GetStyleSetName(), "OpenPype.Logo")
+ );
+ }
+ ToolbarBuilder.EndSection();
+}
+
+
+void FOpenPypeModule::MenuPopup() {
+ UOpenPypePythonBridge* bridge = UOpenPypePythonBridge::Get();
+ bridge->RunInPython_Popup();
+}
+
+void FOpenPypeModule::MenuDialog() {
+ UOpenPypePythonBridge* bridge = UOpenPypePythonBridge::Get();
+ bridge->RunInPython_Dialog();
+}
+
+IMPLEMENT_MODULE(FOpenPypeModule, OpenPype)
diff --git a/openpype/hosts/unreal/integration/UE_4.7/Source/OpenPype/Private/OpenPypeLib.cpp b/openpype/hosts/unreal/integration/UE_4.7/Source/OpenPype/Private/OpenPypeLib.cpp
new file mode 100644
index 00000000000..5facab7b8bd
--- /dev/null
+++ b/openpype/hosts/unreal/integration/UE_4.7/Source/OpenPype/Private/OpenPypeLib.cpp
@@ -0,0 +1,48 @@
+#include "OpenPypeLib.h"
+#include "Misc/Paths.h"
+#include "Misc/ConfigCacheIni.h"
+#include "UObject/UnrealType.h"
+
+/**
+ * Sets color on folder icon on given path
+ * @param InPath - path to folder
+ * @param InFolderColor - color of the folder
+ * @warning This color will appear only after Editor restart. Is there a better way?
+ */
+
+void UOpenPypeLib::CSetFolderColor(FString FolderPath, FLinearColor FolderColor, bool bForceAdd)
+{
+ auto SaveColorInternal = [](FString InPath, FLinearColor InFolderColor)
+ {
+ // Saves the color of the folder to the config
+ if (FPaths::FileExists(GEditorPerProjectIni))
+ {
+ GConfig->SetString(TEXT("PathColor"), *InPath, *InFolderColor.ToString(), GEditorPerProjectIni);
+ }
+
+ };
+
+ SaveColorInternal(FolderPath, FolderColor);
+
+}
+/**
+ * Returns all poperties on given object
+ * @param cls - class
+ * @return TArray of properties
+ */
+TArray UOpenPypeLib::GetAllProperties(UClass* cls)
+{
+ TArray Ret;
+ if (cls != nullptr)
+ {
+ for (TFieldIterator It(cls); It; ++It)
+ {
+ FProperty* Property = *It;
+ if (Property->HasAnyPropertyFlags(EPropertyFlags::CPF_Edit))
+ {
+ Ret.Add(Property->GetName());
+ }
+ }
+ }
+ return Ret;
+}
diff --git a/openpype/hosts/unreal/integration/UE_4.7/Source/OpenPype/Private/OpenPypePublishInstance.cpp b/openpype/hosts/unreal/integration/UE_4.7/Source/OpenPype/Private/OpenPypePublishInstance.cpp
new file mode 100644
index 00000000000..4f1e846c0bf
--- /dev/null
+++ b/openpype/hosts/unreal/integration/UE_4.7/Source/OpenPype/Private/OpenPypePublishInstance.cpp
@@ -0,0 +1,108 @@
+#pragma once
+
+#include "OpenPypePublishInstance.h"
+#include "AssetRegistryModule.h"
+
+
+UOpenPypePublishInstance::UOpenPypePublishInstance(const FObjectInitializer& ObjectInitializer)
+ : UObject(ObjectInitializer)
+{
+ FAssetRegistryModule& AssetRegistryModule = FModuleManager::LoadModuleChecked("AssetRegistry");
+ FString path = UOpenPypePublishInstance::GetPathName();
+ FARFilter Filter;
+ Filter.PackagePaths.Add(FName(*path));
+
+ AssetRegistryModule.Get().OnAssetAdded().AddUObject(this, &UOpenPypePublishInstance::OnAssetAdded);
+ AssetRegistryModule.Get().OnAssetRemoved().AddUObject(this, &UOpenPypePublishInstance::OnAssetRemoved);
+ AssetRegistryModule.Get().OnAssetRenamed().AddUObject(this, &UOpenPypePublishInstance::OnAssetRenamed);
+}
+
+void UOpenPypePublishInstance::OnAssetAdded(const FAssetData& AssetData)
+{
+ TArray split;
+
+ // get directory of current container
+ FString selfFullPath = UOpenPypePublishInstance::GetPathName();
+ FString selfDir = FPackageName::GetLongPackagePath(*selfFullPath);
+
+ // get asset path and class
+ FString assetPath = AssetData.GetFullName();
+ FString assetFName = AssetData.AssetClass.ToString();
+
+ // split path
+ assetPath.ParseIntoArray(split, TEXT(" "), true);
+
+ FString assetDir = FPackageName::GetLongPackagePath(*split[1]);
+
+ // take interest only in paths starting with path of current container
+ if (assetDir.StartsWith(*selfDir))
+ {
+ // exclude self
+ if (assetFName != "OpenPypePublishInstance")
+ {
+ assets.Add(assetPath);
+ UE_LOG(LogTemp, Log, TEXT("%s: asset added to %s"), *selfFullPath, *selfDir);
+ }
+ }
+}
+
+void UOpenPypePublishInstance::OnAssetRemoved(const FAssetData& AssetData)
+{
+ TArray split;
+
+ // get directory of current container
+ FString selfFullPath = UOpenPypePublishInstance::GetPathName();
+ FString selfDir = FPackageName::GetLongPackagePath(*selfFullPath);
+
+ // get asset path and class
+ FString assetPath = AssetData.GetFullName();
+ FString assetFName = AssetData.AssetClass.ToString();
+
+ // split path
+ assetPath.ParseIntoArray(split, TEXT(" "), true);
+
+ FString assetDir = FPackageName::GetLongPackagePath(*split[1]);
+
+ // take interest only in paths starting with path of current container
+ FString path = UOpenPypePublishInstance::GetPathName();
+ FString lpp = FPackageName::GetLongPackagePath(*path);
+
+ if (assetDir.StartsWith(*selfDir))
+ {
+ // exclude self
+ if (assetFName != "OpenPypePublishInstance")
+ {
+ // UE_LOG(LogTemp, Warning, TEXT("%s: asset removed"), *lpp);
+ assets.Remove(assetPath);
+ }
+ }
+}
+
+void UOpenPypePublishInstance::OnAssetRenamed(const FAssetData& AssetData, const FString& str)
+{
+ TArray split;
+
+ // get directory of current container
+ FString selfFullPath = UOpenPypePublishInstance::GetPathName();
+ FString selfDir = FPackageName::GetLongPackagePath(*selfFullPath);
+
+ // get asset path and class
+ FString assetPath = AssetData.GetFullName();
+ FString assetFName = AssetData.AssetClass.ToString();
+
+ // split path
+ assetPath.ParseIntoArray(split, TEXT(" "), true);
+
+ FString assetDir = FPackageName::GetLongPackagePath(*split[1]);
+ if (assetDir.StartsWith(*selfDir))
+ {
+ // exclude self
+ if (assetFName != "AssetContainer")
+ {
+
+ assets.Remove(str);
+ assets.Add(assetPath);
+ // UE_LOG(LogTemp, Warning, TEXT("%s: asset renamed %s"), *lpp, *str);
+ }
+ }
+}
diff --git a/openpype/hosts/unreal/integration/UE_4.7/Source/OpenPype/Private/OpenPypePublishInstanceFactory.cpp b/openpype/hosts/unreal/integration/UE_4.7/Source/OpenPype/Private/OpenPypePublishInstanceFactory.cpp
new file mode 100644
index 00000000000..e61964c6893
--- /dev/null
+++ b/openpype/hosts/unreal/integration/UE_4.7/Source/OpenPype/Private/OpenPypePublishInstanceFactory.cpp
@@ -0,0 +1,20 @@
+#include "OpenPypePublishInstanceFactory.h"
+#include "OpenPypePublishInstance.h"
+
+UOpenPypePublishInstanceFactory::UOpenPypePublishInstanceFactory(const FObjectInitializer& ObjectInitializer)
+ : UFactory(ObjectInitializer)
+{
+ SupportedClass = UOpenPypePublishInstance::StaticClass();
+ bCreateNew = false;
+ bEditorImport = true;
+}
+
+UObject* UOpenPypePublishInstanceFactory::FactoryCreateNew(UClass* Class, UObject* InParent, FName Name, EObjectFlags Flags, UObject* Context, FFeedbackContext* Warn)
+{
+ UOpenPypePublishInstance* OpenPypePublishInstance = NewObject(InParent, Class, Name, Flags);
+ return OpenPypePublishInstance;
+}
+
+bool UOpenPypePublishInstanceFactory::ShouldShowInNewMenu() const {
+ return false;
+}
diff --git a/openpype/hosts/unreal/integration/UE_4.7/Source/OpenPype/Private/OpenPypePythonBridge.cpp b/openpype/hosts/unreal/integration/UE_4.7/Source/OpenPype/Private/OpenPypePythonBridge.cpp
new file mode 100644
index 00000000000..81132315030
--- /dev/null
+++ b/openpype/hosts/unreal/integration/UE_4.7/Source/OpenPype/Private/OpenPypePythonBridge.cpp
@@ -0,0 +1,13 @@
+#include "OpenPypePythonBridge.h"
+
+UOpenPypePythonBridge* UOpenPypePythonBridge::Get()
+{
+ TArray OpenPypePythonBridgeClasses;
+ GetDerivedClasses(UOpenPypePythonBridge::StaticClass(), OpenPypePythonBridgeClasses);
+ int32 NumClasses = OpenPypePythonBridgeClasses.Num();
+ if (NumClasses > 0)
+ {
+ return Cast(OpenPypePythonBridgeClasses[NumClasses - 1]->GetDefaultObject());
+ }
+ return nullptr;
+};
\ No newline at end of file
diff --git a/openpype/hosts/unreal/integration/UE_4.7/Source/OpenPype/Private/OpenPypeStyle.cpp b/openpype/hosts/unreal/integration/UE_4.7/Source/OpenPype/Private/OpenPypeStyle.cpp
new file mode 100644
index 00000000000..a51c2d6aa54
--- /dev/null
+++ b/openpype/hosts/unreal/integration/UE_4.7/Source/OpenPype/Private/OpenPypeStyle.cpp
@@ -0,0 +1,70 @@
+#include "OpenPypeStyle.h"
+#include "Framework/Application/SlateApplication.h"
+#include "Styling/SlateStyle.h"
+#include "Styling/SlateStyleRegistry.h"
+
+
+TUniquePtr< FSlateStyleSet > FOpenPypeStyle::OpenPypeStyleInstance = nullptr;
+
+void FOpenPypeStyle::Initialize()
+{
+ if (!OpenPypeStyleInstance.IsValid())
+ {
+ OpenPypeStyleInstance = Create();
+ FSlateStyleRegistry::RegisterSlateStyle(*OpenPypeStyleInstance);
+ }
+}
+
+void FOpenPypeStyle::Shutdown()
+{
+ if (OpenPypeStyleInstance.IsValid())
+ {
+ FSlateStyleRegistry::UnRegisterSlateStyle(*OpenPypeStyleInstance);
+ OpenPypeStyleInstance.Reset();
+ }
+}
+
+FName FOpenPypeStyle::GetStyleSetName()
+{
+ static FName StyleSetName(TEXT("OpenPypeStyle"));
+ return StyleSetName;
+}
+
+FName FOpenPypeStyle::GetContextName()
+{
+ static FName ContextName(TEXT("OpenPype"));
+ return ContextName;
+}
+
+#define IMAGE_BRUSH(RelativePath, ...) FSlateImageBrush( Style->RootToContentDir( RelativePath, TEXT(".png") ), __VA_ARGS__ )
+
+const FVector2D Icon40x40(40.0f, 40.0f);
+
+TUniquePtr< FSlateStyleSet > FOpenPypeStyle::Create()
+{
+ TUniquePtr< FSlateStyleSet > Style = MakeUnique(GetStyleSetName());
+ Style->SetContentRoot(FPaths::ProjectPluginsDir() / TEXT("OpenPype/Resources"));
+
+ return Style;
+}
+
+void FOpenPypeStyle::SetIcon(const FString& StyleName, const FString& ResourcePath)
+{
+ FSlateStyleSet* Style = OpenPypeStyleInstance.Get();
+
+ FString Name(GetContextName().ToString());
+ Name = Name + "." + StyleName;
+ Style->Set(*Name, new FSlateImageBrush(Style->RootToContentDir(ResourcePath, TEXT(".png")), Icon40x40));
+
+
+ FSlateApplication::Get().GetRenderer()->ReloadTextureResources();
+}
+
+#undef IMAGE_BRUSH
+
+const ISlateStyle& FOpenPypeStyle::Get()
+{
+ check(OpenPypeStyleInstance);
+ return *OpenPypeStyleInstance;
+ return *OpenPypeStyleInstance;
+}
diff --git a/openpype/hosts/unreal/integration/UE_4.7/Source/OpenPype/Public/AssetContainer.h b/openpype/hosts/unreal/integration/UE_4.7/Source/OpenPype/Public/AssetContainer.h
new file mode 100644
index 00000000000..3c2a360c78e
--- /dev/null
+++ b/openpype/hosts/unreal/integration/UE_4.7/Source/OpenPype/Public/AssetContainer.h
@@ -0,0 +1,39 @@
+// Fill out your copyright notice in the Description page of Project Settings.
+
+#pragma once
+
+#include "CoreMinimal.h"
+#include "UObject/NoExportTypes.h"
+#include "Engine/AssetUserData.h"
+#include "AssetData.h"
+#include "AssetContainer.generated.h"
+
+/**
+ *
+ */
+UCLASS(Blueprintable)
+class OPENPYPE_API UAssetContainer : public UAssetUserData
+{
+ GENERATED_BODY()
+
+public:
+
+ UAssetContainer(const FObjectInitializer& ObjectInitalizer);
+ // ~UAssetContainer();
+
+ UPROPERTY(EditAnywhere, BlueprintReadOnly)
+ TArray assets;
+
+ // There seems to be no reflection option to expose array of FAssetData
+ /*
+ UPROPERTY(Transient, BlueprintReadOnly, Category = "Python", meta=(DisplayName="Assets Data"))
+ TArray assetsData;
+ */
+private:
+ TArray assetsData;
+ void OnAssetAdded(const FAssetData& AssetData);
+ void OnAssetRemoved(const FAssetData& AssetData);
+ void OnAssetRenamed(const FAssetData& AssetData, const FString& str);
+};
+
+
diff --git a/openpype/hosts/unreal/integration/UE_4.7/Source/OpenPype/Public/AssetContainerFactory.h b/openpype/hosts/unreal/integration/UE_4.7/Source/OpenPype/Public/AssetContainerFactory.h
new file mode 100644
index 00000000000..331ce6bb50d
--- /dev/null
+++ b/openpype/hosts/unreal/integration/UE_4.7/Source/OpenPype/Public/AssetContainerFactory.h
@@ -0,0 +1,21 @@
+// Fill out your copyright notice in the Description page of Project Settings.
+
+#pragma once
+
+#include "CoreMinimal.h"
+#include "Factories/Factory.h"
+#include "AssetContainerFactory.generated.h"
+
+/**
+ *
+ */
+UCLASS()
+class OPENPYPE_API UAssetContainerFactory : public UFactory
+{
+ GENERATED_BODY()
+
+public:
+ UAssetContainerFactory(const FObjectInitializer& ObjectInitializer);
+ virtual UObject* FactoryCreateNew(UClass* Class, UObject* InParent, FName Name, EObjectFlags Flags, UObject* Context, FFeedbackContext* Warn) override;
+ virtual bool ShouldShowInNewMenu() const override;
+};
\ No newline at end of file
diff --git a/openpype/hosts/unreal/integration/UE_4.7/Source/OpenPype/Public/OpenPype.h b/openpype/hosts/unreal/integration/UE_4.7/Source/OpenPype/Public/OpenPype.h
new file mode 100644
index 00000000000..db3f2993548
--- /dev/null
+++ b/openpype/hosts/unreal/integration/UE_4.7/Source/OpenPype/Public/OpenPype.h
@@ -0,0 +1,21 @@
+// Copyright 1998-2019 Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include "Engine.h"
+
+
+class FOpenPypeModule : public IModuleInterface
+{
+public:
+ virtual void StartupModule() override;
+ virtual void ShutdownModule() override;
+
+private:
+
+ void AddMenuEntry(FMenuBuilder& MenuBuilder);
+ void AddToobarEntry(FToolBarBuilder& ToolbarBuilder);
+ void MenuPopup();
+ void MenuDialog();
+
+};
diff --git a/openpype/hosts/unreal/integration/UE_4.7/Source/OpenPype/Public/OpenPypeLib.h b/openpype/hosts/unreal/integration/UE_4.7/Source/OpenPype/Public/OpenPypeLib.h
new file mode 100644
index 00000000000..59e9c8bd768
--- /dev/null
+++ b/openpype/hosts/unreal/integration/UE_4.7/Source/OpenPype/Public/OpenPypeLib.h
@@ -0,0 +1,19 @@
+#pragma once
+
+#include "Engine.h"
+#include "OpenPypeLib.generated.h"
+
+
+UCLASS(Blueprintable)
+class OPENPYPE_API UOpenPypeLib : public UObject
+{
+
+ GENERATED_BODY()
+
+public:
+ UFUNCTION(BlueprintCallable, Category = Python)
+ static void CSetFolderColor(FString FolderPath, FLinearColor FolderColor, bool bForceAdd);
+
+ UFUNCTION(BlueprintCallable, Category = Python)
+ static TArray GetAllProperties(UClass* cls);
+};
\ No newline at end of file
diff --git a/openpype/hosts/unreal/integration/UE_4.7/Source/OpenPype/Public/OpenPypePublishInstance.h b/openpype/hosts/unreal/integration/UE_4.7/Source/OpenPype/Public/OpenPypePublishInstance.h
new file mode 100644
index 00000000000..0a27a078d7a
--- /dev/null
+++ b/openpype/hosts/unreal/integration/UE_4.7/Source/OpenPype/Public/OpenPypePublishInstance.h
@@ -0,0 +1,21 @@
+#pragma once
+
+#include "Engine.h"
+#include "OpenPypePublishInstance.generated.h"
+
+
+UCLASS(Blueprintable)
+class OPENPYPE_API UOpenPypePublishInstance : public UObject
+{
+ GENERATED_BODY()
+
+public:
+ UOpenPypePublishInstance(const FObjectInitializer& ObjectInitalizer);
+
+ UPROPERTY(EditAnywhere, BlueprintReadOnly)
+ TArray assets;
+private:
+ void OnAssetAdded(const FAssetData& AssetData);
+ void OnAssetRemoved(const FAssetData& AssetData);
+ void OnAssetRenamed(const FAssetData& AssetData, const FString& str);
+};
\ No newline at end of file
diff --git a/openpype/hosts/unreal/integration/UE_4.7/Source/OpenPype/Public/OpenPypePublishInstanceFactory.h b/openpype/hosts/unreal/integration/UE_4.7/Source/OpenPype/Public/OpenPypePublishInstanceFactory.h
new file mode 100644
index 00000000000..a2b3abe13ed
--- /dev/null
+++ b/openpype/hosts/unreal/integration/UE_4.7/Source/OpenPype/Public/OpenPypePublishInstanceFactory.h
@@ -0,0 +1,19 @@
+#pragma once
+
+#include "CoreMinimal.h"
+#include "Factories/Factory.h"
+#include "OpenPypePublishInstanceFactory.generated.h"
+
+/**
+ *
+ */
+UCLASS()
+class OPENPYPE_API UOpenPypePublishInstanceFactory : public UFactory
+{
+ GENERATED_BODY()
+
+public:
+ UOpenPypePublishInstanceFactory(const FObjectInitializer& ObjectInitializer);
+ virtual UObject* FactoryCreateNew(UClass* Class, UObject* InParent, FName Name, EObjectFlags Flags, UObject* Context, FFeedbackContext* Warn) override;
+ virtual bool ShouldShowInNewMenu() const override;
+};
\ No newline at end of file
diff --git a/openpype/hosts/unreal/integration/UE_4.7/Source/OpenPype/Public/OpenPypePythonBridge.h b/openpype/hosts/unreal/integration/UE_4.7/Source/OpenPype/Public/OpenPypePythonBridge.h
new file mode 100644
index 00000000000..692aab2e5e8
--- /dev/null
+++ b/openpype/hosts/unreal/integration/UE_4.7/Source/OpenPype/Public/OpenPypePythonBridge.h
@@ -0,0 +1,20 @@
+#pragma once
+#include "Engine.h"
+#include "OpenPypePythonBridge.generated.h"
+
+UCLASS(Blueprintable)
+class UOpenPypePythonBridge : public UObject
+{
+ GENERATED_BODY()
+
+public:
+ UFUNCTION(BlueprintCallable, Category = Python)
+ static UOpenPypePythonBridge* Get();
+
+ UFUNCTION(BlueprintImplementableEvent, Category = Python)
+ void RunInPython_Popup() const;
+
+ UFUNCTION(BlueprintImplementableEvent, Category = Python)
+ void RunInPython_Dialog() const;
+
+};
diff --git a/openpype/hosts/unreal/integration/UE_4.7/Source/OpenPype/Public/OpenPypeStyle.h b/openpype/hosts/unreal/integration/UE_4.7/Source/OpenPype/Public/OpenPypeStyle.h
new file mode 100644
index 00000000000..fbc8bcdd5b3
--- /dev/null
+++ b/openpype/hosts/unreal/integration/UE_4.7/Source/OpenPype/Public/OpenPypeStyle.h
@@ -0,0 +1,22 @@
+#pragma once
+#include "CoreMinimal.h"
+
+class FSlateStyleSet;
+class ISlateStyle;
+
+
+class FOpenPypeStyle
+{
+public:
+ static void Initialize();
+ static void Shutdown();
+ static const ISlateStyle& Get();
+ static FName GetStyleSetName();
+ static FName GetContextName();
+
+ static void SetIcon(const FString& StyleName, const FString& ResourcePath);
+
+private:
+ static TUniquePtr< FSlateStyleSet > Create();
+ static TUniquePtr< FSlateStyleSet > OpenPypeStyleInstance;
+};
\ No newline at end of file
diff --git a/openpype/hosts/unreal/integration/UE_5.0/.gitignore b/openpype/hosts/unreal/integration/UE_5.0/.gitignore
new file mode 100644
index 00000000000..b32a6f55e5d
--- /dev/null
+++ b/openpype/hosts/unreal/integration/UE_5.0/.gitignore
@@ -0,0 +1,35 @@
+# Prerequisites
+*.d
+
+# Compiled Object files
+*.slo
+*.lo
+*.o
+*.obj
+
+# Precompiled Headers
+*.gch
+*.pch
+
+# Compiled Dynamic libraries
+*.so
+*.dylib
+*.dll
+
+# Fortran module files
+*.mod
+*.smod
+
+# Compiled Static libraries
+*.lai
+*.la
+*.a
+*.lib
+
+# Executables
+*.exe
+*.out
+*.app
+
+/Binaries
+/Intermediate
diff --git a/openpype/hosts/unreal/integration/UE_5.0/Content/Python/init_unreal.py b/openpype/hosts/unreal/integration/UE_5.0/Content/Python/init_unreal.py
new file mode 100644
index 00000000000..b85f9706992
--- /dev/null
+++ b/openpype/hosts/unreal/integration/UE_5.0/Content/Python/init_unreal.py
@@ -0,0 +1,30 @@
+import unreal
+
+openpype_detected = True
+try:
+ from openpype.pipeline import install_host
+ from openpype.hosts.unreal.api import UnrealHost
+
+ openpype_host = UnrealHost()
+except ImportError as exc:
+ openpype_host = None
+ openpype_detected = False
+ unreal.log_error("OpenPype: cannot load OpenPype [ {} ]".format(exc))
+
+if openpype_detected:
+ install_host(openpype_host)
+
+
+@unreal.uclass()
+class OpenPypeIntegration(unreal.OpenPypePythonBridge):
+ @unreal.ufunction(override=True)
+ def RunInPython_Popup(self):
+ unreal.log_warning("OpenPype: showing tools popup")
+ if openpype_detected:
+ openpype_host.show_tools_popup()
+
+ @unreal.ufunction(override=True)
+ def RunInPython_Dialog(self):
+ unreal.log_warning("OpenPype: showing tools dialog")
+ if openpype_detected:
+ openpype_host.show_tools_dialog()
diff --git a/openpype/hosts/unreal/integration/UE_5.0/OpenPype.uplugin b/openpype/hosts/unreal/integration/UE_5.0/OpenPype.uplugin
new file mode 100644
index 00000000000..4c7a74403c2
--- /dev/null
+++ b/openpype/hosts/unreal/integration/UE_5.0/OpenPype.uplugin
@@ -0,0 +1,24 @@
+{
+ "FileVersion": 3,
+ "Version": 1,
+ "VersionName": "1.0",
+ "FriendlyName": "OpenPype",
+ "Description": "OpenPype Integration",
+ "Category": "OpenPype.Integration",
+ "CreatedBy": "Ondrej Samohel",
+ "CreatedByURL": "https://openpype.io",
+ "DocsURL": "https://openpype.io/docs/artist_hosts_unreal",
+ "MarketplaceURL": "",
+ "SupportURL": "https://pype.club/",
+ "CanContainContent": true,
+ "IsBetaVersion": true,
+ "IsExperimentalVersion": false,
+ "Installed": false,
+ "Modules": [
+ {
+ "Name": "OpenPype",
+ "Type": "Editor",
+ "LoadingPhase": "Default"
+ }
+ ]
+}
\ No newline at end of file
diff --git a/openpype/hosts/unreal/integration/UE_5.0/README.md b/openpype/hosts/unreal/integration/UE_5.0/README.md
new file mode 100644
index 00000000000..cf0aa622c21
--- /dev/null
+++ b/openpype/hosts/unreal/integration/UE_5.0/README.md
@@ -0,0 +1,11 @@
+# OpenPype Unreal Integration plugin - UE 5.x
+
+This is plugin for Unreal Editor, creating menu for [OpenPype](https://github.com/getavalon) tools to run.
+
+## How does this work
+
+Plugin is creating basic menu items in **Window/OpenPype** section of Unreal Editor main menu and a button
+on the main toolbar with associated menu. Clicking on those menu items is calling callbacks that are
+declared in C++ but needs to be implemented during Unreal Editor
+startup in `Plugins/OpenPype/Content/Python/init_unreal.py` - this should be executed by Unreal Editor
+automatically.
diff --git a/openpype/hosts/unreal/integration/UE_5.0/Resources/openpype128.png b/openpype/hosts/unreal/integration/UE_5.0/Resources/openpype128.png
new file mode 100644
index 00000000000..abe8a807ef4
Binary files /dev/null and b/openpype/hosts/unreal/integration/UE_5.0/Resources/openpype128.png differ
diff --git a/openpype/hosts/unreal/integration/UE_5.0/Resources/openpype40.png b/openpype/hosts/unreal/integration/UE_5.0/Resources/openpype40.png
new file mode 100644
index 00000000000..f983e7a1f20
Binary files /dev/null and b/openpype/hosts/unreal/integration/UE_5.0/Resources/openpype40.png differ
diff --git a/openpype/hosts/unreal/integration/UE_5.0/Resources/openpype512.png b/openpype/hosts/unreal/integration/UE_5.0/Resources/openpype512.png
new file mode 100644
index 00000000000..97c4d4326bc
Binary files /dev/null and b/openpype/hosts/unreal/integration/UE_5.0/Resources/openpype512.png differ
diff --git a/openpype/hosts/unreal/integration/UE_5.0/Source/OpenPype/OpenPype.Build.cs b/openpype/hosts/unreal/integration/UE_5.0/Source/OpenPype/OpenPype.Build.cs
new file mode 100644
index 00000000000..fcfd268234f
--- /dev/null
+++ b/openpype/hosts/unreal/integration/UE_5.0/Source/OpenPype/OpenPype.Build.cs
@@ -0,0 +1,59 @@
+// Copyright 1998-2019 Epic Games, Inc. All Rights Reserved.
+
+using UnrealBuildTool;
+
+public class OpenPype : ModuleRules
+{
+ public OpenPype(ReadOnlyTargetRules Target) : base(Target)
+ {
+ PCHUsage = ModuleRules.PCHUsageMode.UseExplicitOrSharedPCHs;
+
+ PublicIncludePaths.AddRange(
+ new string[] {
+ // ... add public include paths required here ...
+ }
+ );
+
+
+ PrivateIncludePaths.AddRange(
+ new string[] {
+ // ... add other private include paths required here ...
+ }
+ );
+
+
+ PublicDependencyModuleNames.AddRange(
+ new string[]
+ {
+ "Core",
+ // ... add other public dependencies that you statically link with here ...
+ }
+ );
+
+
+ PrivateDependencyModuleNames.AddRange(
+ new string[]
+ {
+ "Projects",
+ "InputCore",
+ "EditorFramework",
+ "UnrealEd",
+ "ToolMenus",
+ "LevelEditor",
+ "CoreUObject",
+ "Engine",
+ "Slate",
+ "SlateCore",
+ // ... add private dependencies that you statically link with here ...
+ }
+ );
+
+
+ DynamicallyLoadedModuleNames.AddRange(
+ new string[]
+ {
+ // ... add any modules that your module loads dynamically here ...
+ }
+ );
+ }
+}
diff --git a/openpype/hosts/unreal/integration/UE_5.0/Source/OpenPype/Private/AssetContainer.cpp b/openpype/hosts/unreal/integration/UE_5.0/Source/OpenPype/Private/AssetContainer.cpp
new file mode 100644
index 00000000000..c766f87a8ef
--- /dev/null
+++ b/openpype/hosts/unreal/integration/UE_5.0/Source/OpenPype/Private/AssetContainer.cpp
@@ -0,0 +1,115 @@
+// Fill out your copyright notice in the Description page of Project Settings.
+
+#include "AssetContainer.h"
+#include "AssetRegistryModule.h"
+#include "Misc/PackageName.h"
+#include "Engine.h"
+#include "Containers/UnrealString.h"
+
+UAssetContainer::UAssetContainer(const FObjectInitializer& ObjectInitializer)
+: UAssetUserData(ObjectInitializer)
+{
+ FAssetRegistryModule& AssetRegistryModule = FModuleManager::LoadModuleChecked("AssetRegistry");
+ FString path = UAssetContainer::GetPathName();
+ UE_LOG(LogTemp, Warning, TEXT("UAssetContainer %s"), *path);
+ FARFilter Filter;
+ Filter.PackagePaths.Add(FName(*path));
+
+ AssetRegistryModule.Get().OnAssetAdded().AddUObject(this, &UAssetContainer::OnAssetAdded);
+ AssetRegistryModule.Get().OnAssetRemoved().AddUObject(this, &UAssetContainer::OnAssetRemoved);
+ AssetRegistryModule.Get().OnAssetRenamed().AddUObject(this, &UAssetContainer::OnAssetRenamed);
+}
+
+void UAssetContainer::OnAssetAdded(const FAssetData& AssetData)
+{
+ TArray split;
+
+ // get directory of current container
+ FString selfFullPath = UAssetContainer::GetPathName();
+ FString selfDir = FPackageName::GetLongPackagePath(*selfFullPath);
+
+ // get asset path and class
+ FString assetPath = AssetData.GetFullName();
+ FString assetFName = AssetData.AssetClass.ToString();
+
+ // split path
+ assetPath.ParseIntoArray(split, TEXT(" "), true);
+
+ FString assetDir = FPackageName::GetLongPackagePath(*split[1]);
+
+ // take interest only in paths starting with path of current container
+ if (assetDir.StartsWith(*selfDir))
+ {
+ // exclude self
+ if (assetFName != "AssetContainer")
+ {
+ assets.Add(assetPath);
+ assetsData.Add(AssetData);
+ UE_LOG(LogTemp, Log, TEXT("%s: asset added to %s"), *selfFullPath, *selfDir);
+ }
+ }
+}
+
+void UAssetContainer::OnAssetRemoved(const FAssetData& AssetData)
+{
+ TArray split;
+
+ // get directory of current container
+ FString selfFullPath = UAssetContainer::GetPathName();
+ FString selfDir = FPackageName::GetLongPackagePath(*selfFullPath);
+
+ // get asset path and class
+ FString assetPath = AssetData.GetFullName();
+ FString assetFName = AssetData.AssetClass.ToString();
+
+ // split path
+ assetPath.ParseIntoArray(split, TEXT(" "), true);
+
+ FString assetDir = FPackageName::GetLongPackagePath(*split[1]);
+
+ // take interest only in paths starting with path of current container
+ FString path = UAssetContainer::GetPathName();
+ FString lpp = FPackageName::GetLongPackagePath(*path);
+
+ if (assetDir.StartsWith(*selfDir))
+ {
+ // exclude self
+ if (assetFName != "AssetContainer")
+ {
+ // UE_LOG(LogTemp, Warning, TEXT("%s: asset removed"), *lpp);
+ assets.Remove(assetPath);
+ assetsData.Remove(AssetData);
+ }
+ }
+}
+
+void UAssetContainer::OnAssetRenamed(const FAssetData& AssetData, const FString& str)
+{
+ TArray split;
+
+ // get directory of current container
+ FString selfFullPath = UAssetContainer::GetPathName();
+ FString selfDir = FPackageName::GetLongPackagePath(*selfFullPath);
+
+ // get asset path and class
+ FString assetPath = AssetData.GetFullName();
+ FString assetFName = AssetData.AssetClass.ToString();
+
+ // split path
+ assetPath.ParseIntoArray(split, TEXT(" "), true);
+
+ FString assetDir = FPackageName::GetLongPackagePath(*split[1]);
+ if (assetDir.StartsWith(*selfDir))
+ {
+ // exclude self
+ if (assetFName != "AssetContainer")
+ {
+
+ assets.Remove(str);
+ assets.Add(assetPath);
+ assetsData.Remove(AssetData);
+ // UE_LOG(LogTemp, Warning, TEXT("%s: asset renamed %s"), *lpp, *str);
+ }
+ }
+}
+
diff --git a/openpype/hosts/unreal/integration/UE_5.0/Source/OpenPype/Private/AssetContainerFactory.cpp b/openpype/hosts/unreal/integration/UE_5.0/Source/OpenPype/Private/AssetContainerFactory.cpp
new file mode 100644
index 00000000000..b943150bdd0
--- /dev/null
+++ b/openpype/hosts/unreal/integration/UE_5.0/Source/OpenPype/Private/AssetContainerFactory.cpp
@@ -0,0 +1,20 @@
+#include "AssetContainerFactory.h"
+#include "AssetContainer.h"
+
+UAssetContainerFactory::UAssetContainerFactory(const FObjectInitializer& ObjectInitializer)
+ : UFactory(ObjectInitializer)
+{
+ SupportedClass = UAssetContainer::StaticClass();
+ bCreateNew = false;
+ bEditorImport = true;
+}
+
+UObject* UAssetContainerFactory::FactoryCreateNew(UClass* Class, UObject* InParent, FName Name, EObjectFlags Flags, UObject* Context, FFeedbackContext* Warn)
+{
+ UAssetContainer* AssetContainer = NewObject(InParent, Class, Name, Flags);
+ return AssetContainer;
+}
+
+bool UAssetContainerFactory::ShouldShowInNewMenu() const {
+ return false;
+}
diff --git a/openpype/hosts/unreal/integration/UE_5.0/Source/OpenPype/Private/OpenPype.cpp b/openpype/hosts/unreal/integration/UE_5.0/Source/OpenPype/Private/OpenPype.cpp
new file mode 100644
index 00000000000..b3bd9a81b31
--- /dev/null
+++ b/openpype/hosts/unreal/integration/UE_5.0/Source/OpenPype/Private/OpenPype.cpp
@@ -0,0 +1,85 @@
+#include "OpenPype.h"
+#include "OpenPypeStyle.h"
+#include "OpenPypeCommands.h"
+#include "OpenPypePythonBridge.h"
+#include "LevelEditor.h"
+#include "Misc/MessageDialog.h"
+#include "ToolMenus.h"
+
+
+static const FName OpenPypeTabName("OpenPype");
+
+#define LOCTEXT_NAMESPACE "FOpenPypeModule"
+
+// This function is triggered when the plugin is staring up
+void FOpenPypeModule::StartupModule()
+{
+ FOpenPypeStyle::Initialize();
+ FOpenPypeStyle::ReloadTextures();
+ FOpenPypeCommands::Register();
+
+ PluginCommands = MakeShareable(new FUICommandList);
+
+ PluginCommands->MapAction(
+ FOpenPypeCommands::Get().OpenPypeTools,
+ FExecuteAction::CreateRaw(this, &FOpenPypeModule::MenuPopup),
+ FCanExecuteAction());
+ PluginCommands->MapAction(
+ FOpenPypeCommands::Get().OpenPypeToolsDialog,
+ FExecuteAction::CreateRaw(this, &FOpenPypeModule::MenuDialog),
+ FCanExecuteAction());
+
+ UToolMenus::RegisterStartupCallback(FSimpleMulticastDelegate::FDelegate::CreateRaw(this, &FOpenPypeModule::RegisterMenus));
+}
+
+void FOpenPypeModule::ShutdownModule()
+{
+ UToolMenus::UnRegisterStartupCallback(this);
+
+ UToolMenus::UnregisterOwner(this);
+
+ FOpenPypeStyle::Shutdown();
+
+ FOpenPypeCommands::Unregister();
+}
+
+void FOpenPypeModule::RegisterMenus()
+{
+ // Owner will be used for cleanup in call to UToolMenus::UnregisterOwner
+ FToolMenuOwnerScoped OwnerScoped(this);
+
+ {
+ UToolMenu* Menu = UToolMenus::Get()->ExtendMenu("LevelEditor.MainMenu.Tools");
+ {
+ // FToolMenuSection& Section = Menu->FindOrAddSection("OpenPype");
+ FToolMenuSection& Section = Menu->AddSection(
+ "OpenPype",
+ TAttribute(FText::FromString("OpenPype")),
+ FToolMenuInsert("Programming", EToolMenuInsertType::Before)
+ );
+ Section.AddMenuEntryWithCommandList(FOpenPypeCommands::Get().OpenPypeTools, PluginCommands);
+ Section.AddMenuEntryWithCommandList(FOpenPypeCommands::Get().OpenPypeToolsDialog, PluginCommands);
+ }
+ UToolMenu* ToolbarMenu = UToolMenus::Get()->ExtendMenu("LevelEditor.LevelEditorToolBar.PlayToolBar");
+ {
+ FToolMenuSection& Section = ToolbarMenu->FindOrAddSection("PluginTools");
+ {
+ FToolMenuEntry& Entry = Section.AddEntry(FToolMenuEntry::InitToolBarButton(FOpenPypeCommands::Get().OpenPypeTools));
+ Entry.SetCommandList(PluginCommands);
+ }
+ }
+ }
+}
+
+
+void FOpenPypeModule::MenuPopup() {
+ UOpenPypePythonBridge* bridge = UOpenPypePythonBridge::Get();
+ bridge->RunInPython_Popup();
+}
+
+void FOpenPypeModule::MenuDialog() {
+ UOpenPypePythonBridge* bridge = UOpenPypePythonBridge::Get();
+ bridge->RunInPython_Dialog();
+}
+
+IMPLEMENT_MODULE(FOpenPypeModule, OpenPype)
diff --git a/openpype/hosts/unreal/integration/UE_5.0/Source/OpenPype/Private/OpenPypeCommands.cpp b/openpype/hosts/unreal/integration/UE_5.0/Source/OpenPype/Private/OpenPypeCommands.cpp
new file mode 100644
index 00000000000..6187bd7c7ea
--- /dev/null
+++ b/openpype/hosts/unreal/integration/UE_5.0/Source/OpenPype/Private/OpenPypeCommands.cpp
@@ -0,0 +1,13 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include "OpenPypeCommands.h"
+
+#define LOCTEXT_NAMESPACE "FOpenPypeModule"
+
+void FOpenPypeCommands::RegisterCommands()
+{
+ UI_COMMAND(OpenPypeTools, "OpenPype Tools", "Pipeline tools", EUserInterfaceActionType::Button, FInputChord());
+ UI_COMMAND(OpenPypeToolsDialog, "OpenPype Tools Dialog", "Pipeline tools dialog", EUserInterfaceActionType::Button, FInputChord());
+}
+
+#undef LOCTEXT_NAMESPACE
diff --git a/openpype/hosts/unreal/integration/UE_5.0/Source/OpenPype/Private/OpenPypeLib.cpp b/openpype/hosts/unreal/integration/UE_5.0/Source/OpenPype/Private/OpenPypeLib.cpp
new file mode 100644
index 00000000000..5facab7b8bd
--- /dev/null
+++ b/openpype/hosts/unreal/integration/UE_5.0/Source/OpenPype/Private/OpenPypeLib.cpp
@@ -0,0 +1,48 @@
+#include "OpenPypeLib.h"
+#include "Misc/Paths.h"
+#include "Misc/ConfigCacheIni.h"
+#include "UObject/UnrealType.h"
+
+/**
+ * Sets color on folder icon on given path
+ * @param InPath - path to folder
+ * @param InFolderColor - color of the folder
+ * @warning This color will appear only after Editor restart. Is there a better way?
+ */
+
+void UOpenPypeLib::CSetFolderColor(FString FolderPath, FLinearColor FolderColor, bool bForceAdd)
+{
+ auto SaveColorInternal = [](FString InPath, FLinearColor InFolderColor)
+ {
+ // Saves the color of the folder to the config
+ if (FPaths::FileExists(GEditorPerProjectIni))
+ {
+ GConfig->SetString(TEXT("PathColor"), *InPath, *InFolderColor.ToString(), GEditorPerProjectIni);
+ }
+
+ };
+
+ SaveColorInternal(FolderPath, FolderColor);
+
+}
+/**
+ * Returns all poperties on given object
+ * @param cls - class
+ * @return TArray of properties
+ */
+TArray UOpenPypeLib::GetAllProperties(UClass* cls)
+{
+ TArray Ret;
+ if (cls != nullptr)
+ {
+ for (TFieldIterator It(cls); It; ++It)
+ {
+ FProperty* Property = *It;
+ if (Property->HasAnyPropertyFlags(EPropertyFlags::CPF_Edit))
+ {
+ Ret.Add(Property->GetName());
+ }
+ }
+ }
+ return Ret;
+}
diff --git a/openpype/hosts/unreal/integration/UE_5.0/Source/OpenPype/Private/OpenPypePublishInstance.cpp b/openpype/hosts/unreal/integration/UE_5.0/Source/OpenPype/Private/OpenPypePublishInstance.cpp
new file mode 100644
index 00000000000..4f1e846c0bf
--- /dev/null
+++ b/openpype/hosts/unreal/integration/UE_5.0/Source/OpenPype/Private/OpenPypePublishInstance.cpp
@@ -0,0 +1,108 @@
+#pragma once
+
+#include "OpenPypePublishInstance.h"
+#include "AssetRegistryModule.h"
+
+
+UOpenPypePublishInstance::UOpenPypePublishInstance(const FObjectInitializer& ObjectInitializer)
+ : UObject(ObjectInitializer)
+{
+ FAssetRegistryModule& AssetRegistryModule = FModuleManager::LoadModuleChecked("AssetRegistry");
+ FString path = UOpenPypePublishInstance::GetPathName();
+ FARFilter Filter;
+ Filter.PackagePaths.Add(FName(*path));
+
+ AssetRegistryModule.Get().OnAssetAdded().AddUObject(this, &UOpenPypePublishInstance::OnAssetAdded);
+ AssetRegistryModule.Get().OnAssetRemoved().AddUObject(this, &UOpenPypePublishInstance::OnAssetRemoved);
+ AssetRegistryModule.Get().OnAssetRenamed().AddUObject(this, &UOpenPypePublishInstance::OnAssetRenamed);
+}
+
+void UOpenPypePublishInstance::OnAssetAdded(const FAssetData& AssetData)
+{
+ TArray split;
+
+ // get directory of current container
+ FString selfFullPath = UOpenPypePublishInstance::GetPathName();
+ FString selfDir = FPackageName::GetLongPackagePath(*selfFullPath);
+
+ // get asset path and class
+ FString assetPath = AssetData.GetFullName();
+ FString assetFName = AssetData.AssetClass.ToString();
+
+ // split path
+ assetPath.ParseIntoArray(split, TEXT(" "), true);
+
+ FString assetDir = FPackageName::GetLongPackagePath(*split[1]);
+
+ // take interest only in paths starting with path of current container
+ if (assetDir.StartsWith(*selfDir))
+ {
+ // exclude self
+ if (assetFName != "OpenPypePublishInstance")
+ {
+ assets.Add(assetPath);
+ UE_LOG(LogTemp, Log, TEXT("%s: asset added to %s"), *selfFullPath, *selfDir);
+ }
+ }
+}
+
+void UOpenPypePublishInstance::OnAssetRemoved(const FAssetData& AssetData)
+{
+ TArray split;
+
+ // get directory of current container
+ FString selfFullPath = UOpenPypePublishInstance::GetPathName();
+ FString selfDir = FPackageName::GetLongPackagePath(*selfFullPath);
+
+ // get asset path and class
+ FString assetPath = AssetData.GetFullName();
+ FString assetFName = AssetData.AssetClass.ToString();
+
+ // split path
+ assetPath.ParseIntoArray(split, TEXT(" "), true);
+
+ FString assetDir = FPackageName::GetLongPackagePath(*split[1]);
+
+ // take interest only in paths starting with path of current container
+ FString path = UOpenPypePublishInstance::GetPathName();
+ FString lpp = FPackageName::GetLongPackagePath(*path);
+
+ if (assetDir.StartsWith(*selfDir))
+ {
+ // exclude self
+ if (assetFName != "OpenPypePublishInstance")
+ {
+ // UE_LOG(LogTemp, Warning, TEXT("%s: asset removed"), *lpp);
+ assets.Remove(assetPath);
+ }
+ }
+}
+
+void UOpenPypePublishInstance::OnAssetRenamed(const FAssetData& AssetData, const FString& str)
+{
+ TArray split;
+
+ // get directory of current container
+ FString selfFullPath = UOpenPypePublishInstance::GetPathName();
+ FString selfDir = FPackageName::GetLongPackagePath(*selfFullPath);
+
+ // get asset path and class
+ FString assetPath = AssetData.GetFullName();
+ FString assetFName = AssetData.AssetClass.ToString();
+
+ // split path
+ assetPath.ParseIntoArray(split, TEXT(" "), true);
+
+ FString assetDir = FPackageName::GetLongPackagePath(*split[1]);
+ if (assetDir.StartsWith(*selfDir))
+ {
+ // exclude self
+ if (assetFName != "AssetContainer")
+ {
+
+ assets.Remove(str);
+ assets.Add(assetPath);
+ // UE_LOG(LogTemp, Warning, TEXT("%s: asset renamed %s"), *lpp, *str);
+ }
+ }
+}
diff --git a/openpype/hosts/unreal/integration/UE_5.0/Source/OpenPype/Private/OpenPypePublishInstanceFactory.cpp b/openpype/hosts/unreal/integration/UE_5.0/Source/OpenPype/Private/OpenPypePublishInstanceFactory.cpp
new file mode 100644
index 00000000000..e61964c6893
--- /dev/null
+++ b/openpype/hosts/unreal/integration/UE_5.0/Source/OpenPype/Private/OpenPypePublishInstanceFactory.cpp
@@ -0,0 +1,20 @@
+#include "OpenPypePublishInstanceFactory.h"
+#include "OpenPypePublishInstance.h"
+
+UOpenPypePublishInstanceFactory::UOpenPypePublishInstanceFactory(const FObjectInitializer& ObjectInitializer)
+ : UFactory(ObjectInitializer)
+{
+ SupportedClass = UOpenPypePublishInstance::StaticClass();
+ bCreateNew = false;
+ bEditorImport = true;
+}
+
+UObject* UOpenPypePublishInstanceFactory::FactoryCreateNew(UClass* Class, UObject* InParent, FName Name, EObjectFlags Flags, UObject* Context, FFeedbackContext* Warn)
+{
+ UOpenPypePublishInstance* OpenPypePublishInstance = NewObject(InParent, Class, Name, Flags);
+ return OpenPypePublishInstance;
+}
+
+bool UOpenPypePublishInstanceFactory::ShouldShowInNewMenu() const {
+ return false;
+}
diff --git a/openpype/hosts/unreal/integration/UE_5.0/Source/OpenPype/Private/OpenPypePythonBridge.cpp b/openpype/hosts/unreal/integration/UE_5.0/Source/OpenPype/Private/OpenPypePythonBridge.cpp
new file mode 100644
index 00000000000..81132315030
--- /dev/null
+++ b/openpype/hosts/unreal/integration/UE_5.0/Source/OpenPype/Private/OpenPypePythonBridge.cpp
@@ -0,0 +1,13 @@
+#include "OpenPypePythonBridge.h"
+
+UOpenPypePythonBridge* UOpenPypePythonBridge::Get()
+{
+ TArray OpenPypePythonBridgeClasses;
+ GetDerivedClasses(UOpenPypePythonBridge::StaticClass(), OpenPypePythonBridgeClasses);
+ int32 NumClasses = OpenPypePythonBridgeClasses.Num();
+ if (NumClasses > 0)
+ {
+ return Cast(OpenPypePythonBridgeClasses[NumClasses - 1]->GetDefaultObject());
+ }
+ return nullptr;
+};
\ No newline at end of file
diff --git a/openpype/hosts/unreal/integration/UE_5.0/Source/OpenPype/Private/OpenPypeStyle.cpp b/openpype/hosts/unreal/integration/UE_5.0/Source/OpenPype/Private/OpenPypeStyle.cpp
new file mode 100644
index 00000000000..4a53af26b55
--- /dev/null
+++ b/openpype/hosts/unreal/integration/UE_5.0/Source/OpenPype/Private/OpenPypeStyle.cpp
@@ -0,0 +1,61 @@
+#include "OpenPype.h"
+#include "OpenPypeStyle.h"
+#include "Framework/Application/SlateApplication.h"
+#include "Styling/SlateStyleRegistry.h"
+#include "Slate/SlateGameResources.h"
+#include "Interfaces/IPluginManager.h"
+#include "Styling/SlateStyleMacros.h"
+
+#define RootToContentDir Style->RootToContentDir
+
+TSharedPtr FOpenPypeStyle::OpenPypeStyleInstance = nullptr;
+
+void FOpenPypeStyle::Initialize()
+{
+ if (!OpenPypeStyleInstance.IsValid())
+ {
+ OpenPypeStyleInstance = Create();
+ FSlateStyleRegistry::RegisterSlateStyle(*OpenPypeStyleInstance);
+ }
+}
+
+void FOpenPypeStyle::Shutdown()
+{
+ FSlateStyleRegistry::UnRegisterSlateStyle(*OpenPypeStyleInstance);
+ ensure(OpenPypeStyleInstance.IsUnique());
+ OpenPypeStyleInstance.Reset();
+}
+
+FName FOpenPypeStyle::GetStyleSetName()
+{
+ static FName StyleSetName(TEXT("OpenPypeStyle"));
+ return StyleSetName;
+}
+
+const FVector2D Icon16x16(16.0f, 16.0f);
+const FVector2D Icon20x20(20.0f, 20.0f);
+const FVector2D Icon40x40(40.0f, 40.0f);
+
+TSharedRef< FSlateStyleSet > FOpenPypeStyle::Create()
+{
+ TSharedRef< FSlateStyleSet > Style = MakeShareable(new FSlateStyleSet("OpenPypeStyle"));
+ Style->SetContentRoot(IPluginManager::Get().FindPlugin("OpenPype")->GetBaseDir() / TEXT("Resources"));
+
+ Style->Set("OpenPype.OpenPypeTools", new IMAGE_BRUSH(TEXT("openpype40"), Icon40x40));
+ Style->Set("OpenPype.OpenPypeToolsDialog", new IMAGE_BRUSH(TEXT("openpype40"), Icon40x40));
+
+ return Style;
+}
+
+void FOpenPypeStyle::ReloadTextures()
+{
+ if (FSlateApplication::IsInitialized())
+ {
+ FSlateApplication::Get().GetRenderer()->ReloadTextureResources();
+ }
+}
+
+const ISlateStyle& FOpenPypeStyle::Get()
+{
+ return *OpenPypeStyleInstance;
+}
diff --git a/openpype/hosts/unreal/integration/UE_5.0/Source/OpenPype/Public/AssetContainer.h b/openpype/hosts/unreal/integration/UE_5.0/Source/OpenPype/Public/AssetContainer.h
new file mode 100644
index 00000000000..3c2a360c78e
--- /dev/null
+++ b/openpype/hosts/unreal/integration/UE_5.0/Source/OpenPype/Public/AssetContainer.h
@@ -0,0 +1,39 @@
+// Fill out your copyright notice in the Description page of Project Settings.
+
+#pragma once
+
+#include "CoreMinimal.h"
+#include "UObject/NoExportTypes.h"
+#include "Engine/AssetUserData.h"
+#include "AssetData.h"
+#include "AssetContainer.generated.h"
+
+/**
+ *
+ */
+UCLASS(Blueprintable)
+class OPENPYPE_API UAssetContainer : public UAssetUserData
+{
+ GENERATED_BODY()
+
+public:
+
+ UAssetContainer(const FObjectInitializer& ObjectInitalizer);
+ // ~UAssetContainer();
+
+ UPROPERTY(EditAnywhere, BlueprintReadOnly)
+ TArray assets;
+
+ // There seems to be no reflection option to expose array of FAssetData
+ /*
+ UPROPERTY(Transient, BlueprintReadOnly, Category = "Python", meta=(DisplayName="Assets Data"))
+ TArray assetsData;
+ */
+private:
+ TArray assetsData;
+ void OnAssetAdded(const FAssetData& AssetData);
+ void OnAssetRemoved(const FAssetData& AssetData);
+ void OnAssetRenamed(const FAssetData& AssetData, const FString& str);
+};
+
+
diff --git a/openpype/hosts/unreal/integration/UE_5.0/Source/OpenPype/Public/AssetContainerFactory.h b/openpype/hosts/unreal/integration/UE_5.0/Source/OpenPype/Public/AssetContainerFactory.h
new file mode 100644
index 00000000000..331ce6bb50d
--- /dev/null
+++ b/openpype/hosts/unreal/integration/UE_5.0/Source/OpenPype/Public/AssetContainerFactory.h
@@ -0,0 +1,21 @@
+// Fill out your copyright notice in the Description page of Project Settings.
+
+#pragma once
+
+#include "CoreMinimal.h"
+#include "Factories/Factory.h"
+#include "AssetContainerFactory.generated.h"
+
+/**
+ *
+ */
+UCLASS()
+class OPENPYPE_API UAssetContainerFactory : public UFactory
+{
+ GENERATED_BODY()
+
+public:
+ UAssetContainerFactory(const FObjectInitializer& ObjectInitializer);
+ virtual UObject* FactoryCreateNew(UClass* Class, UObject* InParent, FName Name, EObjectFlags Flags, UObject* Context, FFeedbackContext* Warn) override;
+ virtual bool ShouldShowInNewMenu() const override;
+};
\ No newline at end of file
diff --git a/openpype/hosts/unreal/integration/UE_5.0/Source/OpenPype/Public/OpenPype.h b/openpype/hosts/unreal/integration/UE_5.0/Source/OpenPype/Public/OpenPype.h
new file mode 100644
index 00000000000..3ee5eaa65f3
--- /dev/null
+++ b/openpype/hosts/unreal/integration/UE_5.0/Source/OpenPype/Public/OpenPype.h
@@ -0,0 +1,23 @@
+// Copyright 1998-2019 Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include "CoreMinimal.h"
+#include "Modules/ModuleManager.h"
+
+
+class FOpenPypeModule : public IModuleInterface
+{
+public:
+ virtual void StartupModule() override;
+ virtual void ShutdownModule() override;
+
+private:
+ void RegisterMenus();
+
+ void MenuPopup();
+ void MenuDialog();
+
+private:
+ TSharedPtr PluginCommands;
+};
diff --git a/openpype/hosts/unreal/integration/UE_5.0/Source/OpenPype/Public/OpenPypeCommands.h b/openpype/hosts/unreal/integration/UE_5.0/Source/OpenPype/Public/OpenPypeCommands.h
new file mode 100644
index 00000000000..62ffb8de33e
--- /dev/null
+++ b/openpype/hosts/unreal/integration/UE_5.0/Source/OpenPype/Public/OpenPypeCommands.h
@@ -0,0 +1,24 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include "CoreMinimal.h"
+#include "Framework/Commands/Commands.h"
+#include "OpenPypeStyle.h"
+
+class FOpenPypeCommands : public TCommands
+{
+public:
+
+ FOpenPypeCommands()
+ : TCommands(TEXT("OpenPype"), NSLOCTEXT("Contexts", "OpenPype", "OpenPype Tools"), NAME_None, FOpenPypeStyle::GetStyleSetName())
+ {
+ }
+
+ // TCommands<> interface
+ virtual void RegisterCommands() override;
+
+public:
+ TSharedPtr< FUICommandInfo > OpenPypeTools;
+ TSharedPtr< FUICommandInfo > OpenPypeToolsDialog;
+};
diff --git a/openpype/hosts/unreal/integration/UE_5.0/Source/OpenPype/Public/OpenPypeLib.h b/openpype/hosts/unreal/integration/UE_5.0/Source/OpenPype/Public/OpenPypeLib.h
new file mode 100644
index 00000000000..59e9c8bd768
--- /dev/null
+++ b/openpype/hosts/unreal/integration/UE_5.0/Source/OpenPype/Public/OpenPypeLib.h
@@ -0,0 +1,19 @@
+#pragma once
+
+#include "Engine.h"
+#include "OpenPypeLib.generated.h"
+
+
+UCLASS(Blueprintable)
+class OPENPYPE_API UOpenPypeLib : public UObject
+{
+
+ GENERATED_BODY()
+
+public:
+ UFUNCTION(BlueprintCallable, Category = Python)
+ static void CSetFolderColor(FString FolderPath, FLinearColor FolderColor, bool bForceAdd);
+
+ UFUNCTION(BlueprintCallable, Category = Python)
+ static TArray GetAllProperties(UClass* cls);
+};
\ No newline at end of file
diff --git a/openpype/hosts/unreal/integration/UE_5.0/Source/OpenPype/Public/OpenPypePublishInstance.h b/openpype/hosts/unreal/integration/UE_5.0/Source/OpenPype/Public/OpenPypePublishInstance.h
new file mode 100644
index 00000000000..0a27a078d7a
--- /dev/null
+++ b/openpype/hosts/unreal/integration/UE_5.0/Source/OpenPype/Public/OpenPypePublishInstance.h
@@ -0,0 +1,21 @@
+#pragma once
+
+#include "Engine.h"
+#include "OpenPypePublishInstance.generated.h"
+
+
+UCLASS(Blueprintable)
+class OPENPYPE_API UOpenPypePublishInstance : public UObject
+{
+ GENERATED_BODY()
+
+public:
+ UOpenPypePublishInstance(const FObjectInitializer& ObjectInitalizer);
+
+ UPROPERTY(EditAnywhere, BlueprintReadOnly)
+ TArray assets;
+private:
+ void OnAssetAdded(const FAssetData& AssetData);
+ void OnAssetRemoved(const FAssetData& AssetData);
+ void OnAssetRenamed(const FAssetData& AssetData, const FString& str);
+};
\ No newline at end of file
diff --git a/openpype/hosts/unreal/integration/UE_5.0/Source/OpenPype/Public/OpenPypePublishInstanceFactory.h b/openpype/hosts/unreal/integration/UE_5.0/Source/OpenPype/Public/OpenPypePublishInstanceFactory.h
new file mode 100644
index 00000000000..a2b3abe13ed
--- /dev/null
+++ b/openpype/hosts/unreal/integration/UE_5.0/Source/OpenPype/Public/OpenPypePublishInstanceFactory.h
@@ -0,0 +1,19 @@
+#pragma once
+
+#include "CoreMinimal.h"
+#include "Factories/Factory.h"
+#include "OpenPypePublishInstanceFactory.generated.h"
+
+/**
+ *
+ */
+UCLASS()
+class OPENPYPE_API UOpenPypePublishInstanceFactory : public UFactory
+{
+ GENERATED_BODY()
+
+public:
+ UOpenPypePublishInstanceFactory(const FObjectInitializer& ObjectInitializer);
+ virtual UObject* FactoryCreateNew(UClass* Class, UObject* InParent, FName Name, EObjectFlags Flags, UObject* Context, FFeedbackContext* Warn) override;
+ virtual bool ShouldShowInNewMenu() const override;
+};
\ No newline at end of file
diff --git a/openpype/hosts/unreal/integration/UE_5.0/Source/OpenPype/Public/OpenPypePythonBridge.h b/openpype/hosts/unreal/integration/UE_5.0/Source/OpenPype/Public/OpenPypePythonBridge.h
new file mode 100644
index 00000000000..692aab2e5e8
--- /dev/null
+++ b/openpype/hosts/unreal/integration/UE_5.0/Source/OpenPype/Public/OpenPypePythonBridge.h
@@ -0,0 +1,20 @@
+#pragma once
+#include "Engine.h"
+#include "OpenPypePythonBridge.generated.h"
+
+UCLASS(Blueprintable)
+class UOpenPypePythonBridge : public UObject
+{
+ GENERATED_BODY()
+
+public:
+ UFUNCTION(BlueprintCallable, Category = Python)
+ static UOpenPypePythonBridge* Get();
+
+ UFUNCTION(BlueprintImplementableEvent, Category = Python)
+ void RunInPython_Popup() const;
+
+ UFUNCTION(BlueprintImplementableEvent, Category = Python)
+ void RunInPython_Dialog() const;
+
+};
diff --git a/openpype/hosts/unreal/integration/UE_5.0/Source/OpenPype/Public/OpenPypeStyle.h b/openpype/hosts/unreal/integration/UE_5.0/Source/OpenPype/Public/OpenPypeStyle.h
new file mode 100644
index 00000000000..ae704251e11
--- /dev/null
+++ b/openpype/hosts/unreal/integration/UE_5.0/Source/OpenPype/Public/OpenPypeStyle.h
@@ -0,0 +1,18 @@
+#pragma once
+#include "CoreMinimal.h"
+#include "Styling/SlateStyle.h"
+
+class FOpenPypeStyle
+{
+public:
+ static void Initialize();
+ static void Shutdown();
+ static void ReloadTextures();
+ static const ISlateStyle& Get();
+ static FName GetStyleSetName();
+
+
+private:
+ static TSharedRef< class FSlateStyleSet > Create();
+ static TSharedPtr< class FSlateStyleSet > OpenPypeStyleInstance;
+};
\ No newline at end of file
diff --git a/openpype/hosts/unreal/lib.py b/openpype/hosts/unreal/lib.py
new file mode 100644
index 00000000000..d02c6de357f
--- /dev/null
+++ b/openpype/hosts/unreal/lib.py
@@ -0,0 +1,479 @@
+# -*- coding: utf-8 -*-
+"""Unreal launching and project tools."""
+
+import os
+import platform
+import json
+from distutils import dir_util
+import subprocess
+import re
+from pathlib import Path
+from collections import OrderedDict
+from openpype.settings import get_project_settings
+
+
+def get_engine_versions(env=None):
+ """Detect Unreal Engine versions.
+
+ This will try to detect location and versions of installed Unreal Engine.
+ Location can be overridden by `UNREAL_ENGINE_LOCATION` environment
+ variable.
+
+ Args:
+ env (dict, optional): Environment to use.
+
+ Returns:
+ OrderedDict: dictionary with version as a key and dir as value.
+ so the highest version is first.
+
+ Example:
+ >>> get_engine_versions()
+ {
+ "4.23": "C:/Epic Games/UE_4.23",
+ "4.24": "C:/Epic Games/UE_4.24"
+ }
+
+ """
+ env = env or os.environ
+ engine_locations = {}
+ try:
+ root, dirs, _ = next(os.walk(env["UNREAL_ENGINE_LOCATION"]))
+
+ for directory in dirs:
+ if directory.startswith("UE"):
+ try:
+ ver = re.split(r"[-_]", directory)[1]
+ except IndexError:
+ continue
+ engine_locations[ver] = os.path.join(root, directory)
+ except KeyError:
+ # environment variable not set
+ pass
+ except OSError:
+ # specified directory doesn't exists
+ pass
+
+ # if we've got something, terminate auto-detection process
+ if engine_locations:
+ return OrderedDict(sorted(engine_locations.items()))
+
+ # else kick in platform specific detection
+ if platform.system().lower() == "windows":
+ return OrderedDict(sorted(_win_get_engine_versions().items()))
+ if platform.system().lower() == "linux":
+ # on linux, there is no installation and getting Unreal Engine involves
+ # git clone. So we'll probably depend on `UNREAL_ENGINE_LOCATION`.
+ pass
+ if platform.system().lower() == "darwin":
+ return OrderedDict(sorted(_darwin_get_engine_version().items()))
+
+ return OrderedDict()
+
+
+def get_editor_executable_path(engine_path: Path, engine_version: str) -> Path:
+ """Get UE Editor executable path."""
+ ue_path = engine_path / "Engine/Binaries"
+ if platform.system().lower() == "windows":
+ if engine_version.split(".")[0] == "4":
+ ue_path /= "Win64/UE4Editor.exe"
+ elif engine_version.split(".")[0] == "5":
+ ue_path /= "Win64/UnrealEditor.exe"
+
+ elif platform.system().lower() == "linux":
+ ue_path /= "Linux/UE4Editor"
+
+ elif platform.system().lower() == "darwin":
+ ue_path /= "Mac/UE4Editor"
+
+ return ue_path
+
+
+def _win_get_engine_versions():
+ """Get Unreal Engine versions on Windows.
+
+ If engines are installed via Epic Games Launcher then there is:
+ `%PROGRAMDATA%/Epic/UnrealEngineLauncher/LauncherInstalled.dat`
+ This file is JSON file listing installed stuff, Unreal engines
+ are marked with `"AppName" = "UE_X.XX"`` like `UE_4.24`
+
+ Returns:
+ dict: version as a key and path as a value.
+
+ """
+ install_json_path = os.path.join(
+ os.getenv("PROGRAMDATA"),
+ "Epic",
+ "UnrealEngineLauncher",
+ "LauncherInstalled.dat",
+ )
+
+ return _parse_launcher_locations(install_json_path)
+
+
+def _darwin_get_engine_version() -> dict:
+ """Get Unreal Engine versions on MacOS.
+
+ It works the same as on Windows, just JSON file location is different.
+
+ Returns:
+ dict: version as a key and path as a value.
+
+ See Also:
+ :func:`_win_get_engine_versions`.
+
+ """
+ install_json_path = os.path.join(
+ os.getenv("HOME"),
+ "Library",
+ "Application Support",
+ "Epic",
+ "UnrealEngineLauncher",
+ "LauncherInstalled.dat",
+ )
+
+ return _parse_launcher_locations(install_json_path)
+
+
+def _parse_launcher_locations(install_json_path: str) -> dict:
+ """This will parse locations from json file.
+
+ Args:
+ install_json_path (str): Path to `LauncherInstalled.dat`.
+
+ Returns:
+ dict: with unreal engine versions as keys and
+ paths to those engine installations as value.
+
+ """
+ engine_locations = {}
+ if os.path.isfile(install_json_path):
+ with open(install_json_path, "r") as ilf:
+ try:
+ install_data = json.load(ilf)
+ except json.JSONDecodeError as e:
+ raise Exception(
+ "Invalid `LauncherInstalled.dat file. `"
+ "Cannot determine Unreal Engine location."
+ ) from e
+
+ for installation in install_data.get("InstallationList", []):
+ if installation.get("AppName").startswith("UE_"):
+ ver = installation.get("AppName").split("_")[1]
+ engine_locations[ver] = installation.get("InstallLocation")
+
+ return engine_locations
+
+
+def create_unreal_project(project_name: str,
+ ue_version: str,
+ pr_dir: Path,
+ engine_path: Path,
+ dev_mode: bool = False,
+ env: dict = None) -> None:
+ """This will create `.uproject` file at specified location.
+
+ As there is no way I know to create a project via command line, this is
+ easiest option. Unreal project file is basically a JSON file. If we find
+ the `OPENPYPE_UNREAL_PLUGIN` environment variable we assume this is the
+ location of the Integration Plugin and we copy its content to the project
+ folder and enable this plugin.
+
+ Args:
+ project_name (str): Name of the project.
+ ue_version (str): Unreal engine version (like 4.23).
+ pr_dir (Path): Path to directory where project will be created.
+ engine_path (Path): Path to Unreal Engine installation.
+ dev_mode (bool, optional): Flag to trigger C++ style Unreal project
+ needing Visual Studio and other tools to compile plugins from
+ sources. This will trigger automatically if `Binaries`
+ directory is not found in plugin folders as this indicates
+ this is only source distribution of the plugin. Dev mode
+ is also set by preset file `unreal/project_setup.json` in
+ **OPENPYPE_CONFIG**.
+ env (dict, optional): Environment to use. If not set, `os.environ`.
+
+ Throws:
+ NotImplementedError: For unsupported platforms.
+
+ Returns:
+ None
+
+ """
+ env = env or os.environ
+ preset = get_project_settings(project_name)["unreal"]["project_setup"]
+ ue_id = ".".join(ue_version.split(".")[:2])
+ # get unreal engine identifier
+ # -------------------------------------------------------------------------
+ # FIXME (antirotor): As of 4.26 this is problem with UE4 built from
+ # sources. In that case Engine ID is calculated per machine/user and not
+ # from Engine files as this code then reads. This then prevents UE4
+ # to directly open project as it will complain about project being
+ # created in different UE4 version. When user convert such project
+ # to his UE4 version, Engine ID is replaced in uproject file. If some
+ # other user tries to open it, it will present him with similar error.
+ ue_modules = Path()
+ if platform.system().lower() == "windows":
+ ue_modules_path = engine_path / "Engine/Binaries/Win64"
+ if ue_version.split(".")[0] == "4":
+ ue_modules_path /= "UE4Editor.modules"
+ elif ue_version.split(".")[0] == "5":
+ ue_modules_path /= "UnrealEditor.modules"
+ ue_modules = Path(ue_modules_path)
+
+ if platform.system().lower() == "linux":
+ ue_modules = Path(os.path.join(engine_path, "Engine", "Binaries",
+ "Linux", "UE4Editor.modules"))
+
+ if platform.system().lower() == "darwin":
+ ue_modules = Path(os.path.join(engine_path, "Engine", "Binaries",
+ "Mac", "UE4Editor.modules"))
+
+ if ue_modules.exists():
+ print("--- Loading Engine ID from modules file ...")
+ with open(ue_modules, "r") as mp:
+ loaded_modules = json.load(mp)
+
+ if loaded_modules.get("BuildId"):
+ ue_id = "{" + loaded_modules.get("BuildId") + "}"
+
+ plugins_path = None
+ if os.path.isdir(env.get("OPENPYPE_UNREAL_PLUGIN", "")):
+ # copy plugin to correct path under project
+ plugins_path = pr_dir / "Plugins"
+ openpype_plugin_path = plugins_path / "OpenPype"
+ if not openpype_plugin_path.is_dir():
+ openpype_plugin_path.mkdir(parents=True, exist_ok=True)
+ dir_util._path_created = {}
+ dir_util.copy_tree(os.environ.get("OPENPYPE_UNREAL_PLUGIN"),
+ openpype_plugin_path.as_posix())
+
+ if not (openpype_plugin_path / "Binaries").is_dir() \
+ or not (openpype_plugin_path / "Intermediate").is_dir():
+ dev_mode = True
+
+ # data for project file
+ data = {
+ "FileVersion": 3,
+ "EngineAssociation": ue_id,
+ "Category": "",
+ "Description": "",
+ "Plugins": [
+ {"Name": "PythonScriptPlugin", "Enabled": True},
+ {"Name": "EditorScriptingUtilities", "Enabled": True},
+ {"Name": "SequencerScripting", "Enabled": True},
+ {"Name": "MovieRenderPipeline", "Enabled": True},
+ {"Name": "OpenPype", "Enabled": True}
+ ]
+ }
+
+ if dev_mode or preset["dev_mode"]:
+ # this will add the project module and necessary source file to
+ # make it a C++ project and to (hopefully) make Unreal Editor to
+ # compile all # sources at start
+
+ data["Modules"] = [{
+ "Name": project_name,
+ "Type": "Runtime",
+ "LoadingPhase": "Default",
+ "AdditionalDependencies": ["Engine"],
+ }]
+
+ # write project file
+ project_file = pr_dir / f"{project_name}.uproject"
+ with open(project_file, mode="w") as pf:
+ json.dump(data, pf, indent=4)
+
+ # ensure we have PySide2 installed in engine
+ python_path = None
+ if platform.system().lower() == "windows":
+ python_path = engine_path / ("Engine/Binaries/ThirdParty/"
+ "Python3/Win64/python.exe")
+
+ if platform.system().lower() == "linux":
+ python_path = engine_path / ("Engine/Binaries/ThirdParty/"
+ "Python3/Linux/bin/python3")
+
+ if platform.system().lower() == "darwin":
+ python_path = engine_path / ("Engine/Binaries/ThirdParty/"
+ "Python3/Mac/bin/python3")
+
+ if not python_path:
+ raise NotImplementedError("Unsupported platform")
+ if not python_path.exists():
+ raise RuntimeError(f"Unreal Python not found at {python_path}")
+ subprocess.check_call(
+ [python_path.as_posix(), "-m", "pip", "install", "pyside2"])
+
+ if dev_mode or preset["dev_mode"]:
+ _prepare_cpp_project(project_file, engine_path, ue_version)
+
+
+def _prepare_cpp_project(
+ project_file: Path, engine_path: Path, ue_version: str) -> None:
+ """Prepare CPP Unreal Project.
+
+ This function will add source files needed for project to be
+ rebuild along with the OpenPype integration plugin.
+
+ There seems not to be automated way to do it from command line.
+ But there might be way to create at least those target and build files
+ by some generator. This needs more research as manually writing
+ those files is rather hackish. :skull_and_crossbones:
+
+
+ Args:
+ project_file (str): Path to .uproject file.
+ engine_path (str): Path to unreal engine associated with project.
+
+ """
+ project_name = project_file.stem
+ project_dir = project_file.parent
+ targets_dir = project_dir / "Source"
+ sources_dir = targets_dir / project_name
+
+ sources_dir.mkdir(parents=True, exist_ok=True)
+ (project_dir / "Content").mkdir(parents=True, exist_ok=True)
+
+ module_target = '''
+using UnrealBuildTool;
+using System.Collections.Generic;
+
+public class {0}Target : TargetRules
+{{
+ public {0}Target( TargetInfo Target) : base(Target)
+ {{
+ Type = TargetType.Game;
+ ExtraModuleNames.AddRange( new string[] {{ "{0}" }} );
+ }}
+}}
+'''.format(project_name)
+
+ editor_module_target = '''
+using UnrealBuildTool;
+using System.Collections.Generic;
+
+public class {0}EditorTarget : TargetRules
+{{
+ public {0}EditorTarget( TargetInfo Target) : base(Target)
+ {{
+ Type = TargetType.Editor;
+
+ ExtraModuleNames.AddRange( new string[] {{ "{0}" }} );
+ }}
+}}
+'''.format(project_name)
+
+ module_build = '''
+using UnrealBuildTool;
+public class {0} : ModuleRules
+{{
+ public {0}(ReadOnlyTargetRules Target) : base(Target)
+ {{
+ PCHUsage = PCHUsageMode.UseExplicitOrSharedPCHs;
+ PublicDependencyModuleNames.AddRange(new string[] {{ "Core",
+ "CoreUObject", "Engine", "InputCore" }});
+ PrivateDependencyModuleNames.AddRange(new string[] {{ }});
+ }}
+}}
+'''.format(project_name)
+
+ module_cpp = '''
+#include "{0}.h"
+#include "Modules/ModuleManager.h"
+
+IMPLEMENT_PRIMARY_GAME_MODULE( FDefaultGameModuleImpl, {0}, "{0}" );
+'''.format(project_name)
+
+ module_header = '''
+#pragma once
+#include "CoreMinimal.h"
+'''
+
+ game_mode_cpp = '''
+#include "{0}GameModeBase.h"
+'''.format(project_name)
+
+ game_mode_h = '''
+#pragma once
+
+#include "CoreMinimal.h"
+#include "GameFramework/GameModeBase.h"
+#include "{0}GameModeBase.generated.h"
+
+UCLASS()
+class {1}_API A{0}GameModeBase : public AGameModeBase
+{{
+ GENERATED_BODY()
+}};
+'''.format(project_name, project_name.upper())
+
+ with open(targets_dir / f"{project_name}.Target.cs", mode="w") as f:
+ f.write(module_target)
+
+ with open(targets_dir / f"{project_name}Editor.Target.cs", mode="w") as f:
+ f.write(editor_module_target)
+
+ with open(sources_dir / f"{project_name}.Build.cs", mode="w") as f:
+ f.write(module_build)
+
+ with open(sources_dir / f"{project_name}.cpp", mode="w") as f:
+ f.write(module_cpp)
+
+ with open(sources_dir / f"{project_name}.h", mode="w") as f:
+ f.write(module_header)
+
+ with open(sources_dir / f"{project_name}GameModeBase.cpp", mode="w") as f:
+ f.write(game_mode_cpp)
+
+ with open(sources_dir / f"{project_name}GameModeBase.h", mode="w") as f:
+ f.write(game_mode_h)
+
+ u_build_tool_path = engine_path / "Engine/Binaries/DotNET"
+ if ue_version.split(".")[0] == "4":
+ u_build_tool_path /= "UnrealBuildTool.exe"
+ elif ue_version.split(".")[0] == "5":
+ u_build_tool_path /= "UnrealBuildTool/UnrealBuildTool.exe"
+ u_build_tool = Path(u_build_tool_path)
+ u_header_tool = None
+
+ arch = "Win64"
+ if platform.system().lower() == "windows":
+ arch = "Win64"
+ u_header_tool = Path(
+ engine_path / "Engine/Binaries/Win64/UnrealHeaderTool.exe")
+ elif platform.system().lower() == "linux":
+ arch = "Linux"
+ u_header_tool = Path(
+ engine_path / "Engine/Binaries/Linux/UnrealHeaderTool")
+ elif platform.system().lower() == "darwin":
+ # we need to test this out
+ arch = "Mac"
+ u_header_tool = Path(
+ engine_path / "Engine/Binaries/Mac/UnrealHeaderTool")
+
+ if not u_header_tool:
+ raise NotImplementedError("Unsupported platform")
+
+ command1 = [u_build_tool.as_posix(), "-projectfiles",
+ f"-project={project_file}", "-progress"]
+
+ subprocess.run(command1)
+
+ command2 = [u_build_tool.as_posix(),
+ f"-ModuleWithSuffix={project_name},3555", arch,
+ "Development", "-TargetType=Editor",
+ f'-Project={project_file}',
+ f'{project_file}',
+ "-IgnoreJunk"]
+
+ subprocess.run(command2)
+
+ """
+ uhtmanifest = os.path.join(os.path.dirname(project_file),
+ f"{project_name}.uhtmanifest")
+
+ command3 = [u_header_tool, f'"{project_file}"', f'"{uhtmanifest}"',
+ "-Unattended", "-WarningsAsErrors", "-installed"]
+
+ subprocess.run(command3)
+ """
diff --git a/openpype/hosts/unreal/plugins/create/create_camera.py b/openpype/hosts/unreal/plugins/create/create_camera.py
index eda2b52be31..28429008340 100644
--- a/openpype/hosts/unreal/plugins/create/create_camera.py
+++ b/openpype/hosts/unreal/plugins/create/create_camera.py
@@ -2,13 +2,11 @@
from unreal import EditorAssetLibrary as eal
from unreal import EditorLevelLibrary as ell
-from openpype.hosts.unreal.api.plugin import Creator
-from avalon.unreal import (
- instantiate,
-)
+from openpype.hosts.unreal.api import plugin
+from openpype.hosts.unreal.api.pipeline import instantiate
-class CreateCamera(Creator):
+class CreateCamera(plugin.Creator):
"""Layout output for character rigs"""
name = "layoutMain"
@@ -16,7 +14,7 @@ class CreateCamera(Creator):
family = "camera"
icon = "cubes"
- root = "/Game/Avalon/Instances"
+ root = "/Game/OpenPype/Instances"
suffix = "_INS"
def __init__(self, *args, **kwargs):
diff --git a/openpype/hosts/unreal/plugins/create/create_layout.py b/openpype/hosts/unreal/plugins/create/create_layout.py
index 239b72787bb..5fef08ce2ac 100644
--- a/openpype/hosts/unreal/plugins/create/create_layout.py
+++ b/openpype/hosts/unreal/plugins/create/create_layout.py
@@ -1,12 +1,12 @@
-from unreal import EditorLevelLibrary as ell
-from openpype.hosts.unreal.api.plugin import Creator
-from avalon.unreal import (
- instantiate,
-)
+# -*- coding: utf-8 -*-
+from unreal import EditorLevelLibrary
+from openpype.hosts.unreal.api import plugin
+from openpype.hosts.unreal.api.pipeline import instantiate
-class CreateLayout(Creator):
- """Layout output for character rigs"""
+
+class CreateLayout(plugin.Creator):
+ """Layout output for character rigs."""
name = "layoutMain"
label = "Layout"
@@ -29,13 +29,13 @@ def process(self):
# sel_objects = unreal.EditorUtilityLibrary.get_selected_assets()
# selection = [a.get_path_name() for a in sel_objects]
- data["level"] = ell.get_editor_world().get_path_name()
+ data["level"] = EditorLevelLibrary.get_editor_world().get_path_name()
data["members"] = []
if (self.options or {}).get("useSelection"):
# Set as members the selected actors
- for actor in ell.get_selected_level_actors():
+ for actor in EditorLevelLibrary.get_selected_level_actors():
data["members"].append("{}.{}".format(
actor.get_outer().get_name(), actor.get_name()))
diff --git a/openpype/hosts/unreal/plugins/create/create_look.py b/openpype/hosts/unreal/plugins/create/create_look.py
index 7d3913b8838..12f6b70ae60 100644
--- a/openpype/hosts/unreal/plugins/create/create_look.py
+++ b/openpype/hosts/unreal/plugins/create/create_look.py
@@ -1,10 +1,11 @@
-import unreal
-from openpype.hosts.unreal.api.plugin import Creator
-from avalon.unreal import pipeline
+# -*- coding: utf-8 -*-
+"""Create look in Unreal."""
+import unreal # noqa
+from openpype.hosts.unreal.api import pipeline, plugin
-class CreateLook(Creator):
- """Shader connections defining shape look"""
+class CreateLook(plugin.Creator):
+ """Shader connections defining shape look."""
name = "unrealLook"
label = "Unreal - Look"
@@ -49,14 +50,14 @@ def process(self):
for material in materials:
name = material.get_editor_property('material_slot_name')
object_path = f"{full_path}/{name}.{name}"
- object = unreal.EditorAssetLibrary.duplicate_loaded_asset(
+ unreal_object = unreal.EditorAssetLibrary.duplicate_loaded_asset(
cube.get_asset(), object_path
)
# Remove the default material of the cube object
- object.get_editor_property('static_materials').pop()
+ unreal_object.get_editor_property('static_materials').pop()
- object.add_material(
+ unreal_object.add_material(
material.get_editor_property('material_interface'))
self.data["members"].append(object_path)
diff --git a/openpype/hosts/unreal/plugins/create/create_render.py b/openpype/hosts/unreal/plugins/create/create_render.py
new file mode 100644
index 00000000000..950799cc108
--- /dev/null
+++ b/openpype/hosts/unreal/plugins/create/create_render.py
@@ -0,0 +1,117 @@
+import unreal
+
+from openpype.hosts.unreal.api import pipeline
+from openpype.hosts.unreal.api.plugin import Creator
+
+
+class CreateRender(Creator):
+ """Create instance for sequence for rendering"""
+
+ name = "unrealRender"
+ label = "Unreal - Render"
+ family = "render"
+ icon = "cube"
+ asset_types = ["LevelSequence"]
+
+ root = "/Game/OpenPype/PublishInstances"
+ suffix = "_INS"
+
+ def process(self):
+ subset = self.data["subset"]
+
+ ar = unreal.AssetRegistryHelpers.get_asset_registry()
+
+ # The asset name is the the third element of the path which contains
+ # the map.
+ # The index of the split path is 3 because the first element is an
+ # empty string, as the path begins with "/Content".
+ a = unreal.EditorUtilityLibrary.get_selected_assets()[0]
+ asset_name = a.get_path_name().split("/")[3]
+
+ # Get the master sequence and the master level.
+ # There should be only one sequence and one level in the directory.
+ filter = unreal.ARFilter(
+ class_names=["LevelSequence"],
+ package_paths=[f"/Game/OpenPype/{asset_name}"],
+ recursive_paths=False)
+ sequences = ar.get_assets(filter)
+ ms = sequences[0].get_editor_property('object_path')
+ filter = unreal.ARFilter(
+ class_names=["World"],
+ package_paths=[f"/Game/OpenPype/{asset_name}"],
+ recursive_paths=False)
+ levels = ar.get_assets(filter)
+ ml = levels[0].get_editor_property('object_path')
+
+ selection = []
+ if (self.options or {}).get("useSelection"):
+ sel_objects = unreal.EditorUtilityLibrary.get_selected_assets()
+ selection = [
+ a.get_path_name() for a in sel_objects
+ if a.get_class().get_name() in self.asset_types]
+ else:
+ selection.append(self.data['sequence'])
+
+ unreal.log(f"selection: {selection}")
+
+ path = f"{self.root}"
+ unreal.EditorAssetLibrary.make_directory(path)
+
+ ar = unreal.AssetRegistryHelpers.get_asset_registry()
+
+ for a in selection:
+ ms_obj = ar.get_asset_by_object_path(ms).get_asset()
+
+ seq_data = None
+
+ if a == ms:
+ seq_data = {
+ "sequence": ms_obj,
+ "output": f"{ms_obj.get_name()}",
+ "frame_range": (
+ ms_obj.get_playback_start(), ms_obj.get_playback_end())
+ }
+ else:
+ seq_data_list = [{
+ "sequence": ms_obj,
+ "output": f"{ms_obj.get_name()}",
+ "frame_range": (
+ ms_obj.get_playback_start(), ms_obj.get_playback_end())
+ }]
+
+ for s in seq_data_list:
+ subscenes = pipeline.get_subsequences(s.get('sequence'))
+
+ for ss in subscenes:
+ curr_data = {
+ "sequence": ss.get_sequence(),
+ "output": (f"{s.get('output')}/"
+ f"{ss.get_sequence().get_name()}"),
+ "frame_range": (
+ ss.get_start_frame(), ss.get_end_frame() - 1)
+ }
+
+ if ss.get_sequence().get_path_name() == a:
+ seq_data = curr_data
+ break
+ seq_data_list.append(curr_data)
+
+ if seq_data is not None:
+ break
+
+ if not seq_data:
+ continue
+
+ d = self.data.copy()
+ d["members"] = [a]
+ d["sequence"] = a
+ d["master_sequence"] = ms
+ d["master_level"] = ml
+ d["output"] = seq_data.get('output')
+ d["frameStart"] = seq_data.get('frame_range')[0]
+ d["frameEnd"] = seq_data.get('frame_range')[1]
+
+ container_name = f"{subset}{self.suffix}"
+ pipeline.create_publish_instance(
+ instance=container_name, path=path)
+ pipeline.imprint(f"{path}/{container_name}", d)
diff --git a/openpype/hosts/unreal/plugins/create/create_staticmeshfbx.py b/openpype/hosts/unreal/plugins/create/create_staticmeshfbx.py
index 4cc67e0f1fa..601c2fae06c 100644
--- a/openpype/hosts/unreal/plugins/create/create_staticmeshfbx.py
+++ b/openpype/hosts/unreal/plugins/create/create_staticmeshfbx.py
@@ -1,12 +1,14 @@
-import unreal
-from openpype.hosts.unreal.api.plugin import Creator
-from avalon.unreal import (
+# -*- coding: utf-8 -*-
+"""Create Static Meshes as FBX geometry."""
+import unreal # noqa
+from openpype.hosts.unreal.api import plugin
+from openpype.hosts.unreal.api.pipeline import (
instantiate,
)
-class CreateStaticMeshFBX(Creator):
- """Static FBX geometry"""
+class CreateStaticMeshFBX(plugin.Creator):
+ """Static FBX geometry."""
name = "unrealStaticMeshMain"
label = "Unreal - Static Mesh"
diff --git a/openpype/hosts/unreal/plugins/load/load_alembic_geometrycache.py b/openpype/hosts/unreal/plugins/load/load_alembic_geometrycache.py
index ad37a7a068c..6ac3531b405 100644
--- a/openpype/hosts/unreal/plugins/load/load_alembic_geometrycache.py
+++ b/openpype/hosts/unreal/plugins/load/load_alembic_geometrycache.py
@@ -1,12 +1,18 @@
+# -*- coding: utf-8 -*-
+"""Loader for published alembics."""
import os
-from avalon import api, pipeline
-from avalon.unreal import lib
-from avalon.unreal import pipeline as unreal_pipeline
-import unreal
+from openpype.pipeline import (
+ get_representation_path,
+ AVALON_CONTAINER_ID
+)
+from openpype.hosts.unreal.api import plugin
+from openpype.hosts.unreal.api import pipeline as unreal_pipeline
+import unreal # noqa
-class PointCacheAlembicLoader(api.Loader):
+
+class PointCacheAlembicLoader(plugin.Loader):
"""Load Point Cache from Alembic"""
families = ["model", "pointcache"]
@@ -56,8 +62,7 @@ def get_task(
return task
def load(self, context, name, namespace, data):
- """
- Load and containerise representation into Content Browser.
+ """Load and containerise representation into Content Browser.
This is two step process. First, import FBX to temporary path and
then call `containerise()` on it - this moves all content to new
@@ -76,10 +81,10 @@ def load(self, context, name, namespace, data):
Returns:
list(str): list of container content
- """
- # Create directory for asset and avalon container
- root = "/Game/Avalon/Assets"
+ """
+ # Create directory for asset and OpenPype container
+ root = "/Game/OpenPype/Assets"
asset = context.get('asset').get('name')
suffix = "_CON"
if asset:
@@ -98,7 +103,7 @@ def load(self, context, name, namespace, data):
frame_start = context.get('asset').get('data').get('frameStart')
frame_end = context.get('asset').get('data').get('frameEnd')
- # If frame start and end are the same, we increse the end frame by
+ # If frame start and end are the same, we increase the end frame by
# one, otherwise Unreal will not import it
if frame_start == frame_end:
frame_end += 1
@@ -109,12 +114,12 @@ def load(self, context, name, namespace, data):
unreal.AssetToolsHelpers.get_asset_tools().import_asset_tasks([task]) # noqa: E501
# Create Asset Container
- lib.create_avalon_container(
+ unreal_pipeline.create_container(
container=container_name, path=asset_dir)
data = {
"schema": "openpype:container-2.0",
- "id": pipeline.AVALON_CONTAINER_ID,
+ "id": AVALON_CONTAINER_ID,
"asset": asset,
"namespace": asset_dir,
"container_name": container_name,
@@ -138,7 +143,7 @@ def load(self, context, name, namespace, data):
def update(self, container, representation):
name = container["asset_name"]
- source_path = api.get_representation_path(representation)
+ source_path = get_representation_path(representation)
destination_path = container["namespace"]
task = self.get_task(source_path, destination_path, name, True)
diff --git a/openpype/hosts/unreal/plugins/load/load_alembic_skeletalmesh.py b/openpype/hosts/unreal/plugins/load/load_alembic_skeletalmesh.py
index b652af0b89b..9fe5f3ab4bb 100644
--- a/openpype/hosts/unreal/plugins/load/load_alembic_skeletalmesh.py
+++ b/openpype/hosts/unreal/plugins/load/load_alembic_skeletalmesh.py
@@ -1,12 +1,17 @@
+# -*- coding: utf-8 -*-
+"""Load Skeletal Mesh alembics."""
import os
-from avalon import api, pipeline
-from avalon.unreal import lib
-from avalon.unreal import pipeline as unreal_pipeline
-import unreal
+from openpype.pipeline import (
+ get_representation_path,
+ AVALON_CONTAINER_ID
+)
+from openpype.hosts.unreal.api import plugin
+from openpype.hosts.unreal.api import pipeline as unreal_pipeline
+import unreal # noqa
-class SkeletalMeshAlembicLoader(api.Loader):
+class SkeletalMeshAlembicLoader(plugin.Loader):
"""Load Unreal SkeletalMesh from Alembic"""
families = ["pointcache"]
@@ -15,9 +20,36 @@ class SkeletalMeshAlembicLoader(api.Loader):
icon = "cube"
color = "orange"
+ def get_task(self, filename, asset_dir, asset_name, replace):
+ task = unreal.AssetImportTask()
+ options = unreal.AbcImportSettings()
+ sm_settings = unreal.AbcStaticMeshSettings()
+ conversion_settings = unreal.AbcConversionSettings(
+ preset=unreal.AbcConversionPreset.CUSTOM,
+ flip_u=False, flip_v=False,
+ rotation=[0.0, 0.0, 0.0],
+ scale=[1.0, 1.0, 1.0])
+
+ task.set_editor_property('filename', filename)
+ task.set_editor_property('destination_path', asset_dir)
+ task.set_editor_property('destination_name', asset_name)
+ task.set_editor_property('replace_existing', replace)
+ task.set_editor_property('automated', True)
+ task.set_editor_property('save', True)
+
+ # set import options here
+ # Unreal 4.24 ignores the settings. It works with Unreal 4.26
+ options.set_editor_property(
+ 'import_type', unreal.AlembicImportType.SKELETAL)
+
+ options.static_mesh_settings = sm_settings
+ options.conversion_settings = conversion_settings
+ task.options = options
+
+ return task
+
def load(self, context, name, namespace, data):
- """
- Load and containerise representation into Content Browser.
+ """Load and containerise representation into Content Browser.
This is two step process. First, import FBX to temporary path and
then call `containerise()` on it - this moves all content to new
@@ -38,48 +70,36 @@ def load(self, context, name, namespace, data):
list(str): list of container content
"""
- # Create directory for asset and avalon container
- root = "/Game/Avalon/Assets"
+ # Create directory for asset and openpype container
+ root = "/Game/OpenPype/Assets"
asset = context.get('asset').get('name')
suffix = "_CON"
if asset:
asset_name = "{}_{}".format(asset, name)
else:
asset_name = "{}".format(name)
+ version = context.get('version').get('name')
tools = unreal.AssetToolsHelpers().get_asset_tools()
asset_dir, container_name = tools.create_unique_asset_name(
- "{}/{}/{}".format(root, asset, name), suffix="")
+ f"{root}/{asset}/{name}_v{version:03d}", suffix="")
container_name += suffix
- unreal.EditorAssetLibrary.make_directory(asset_dir)
-
- task = unreal.AssetImportTask()
-
- task.set_editor_property('filename', self.fname)
- task.set_editor_property('destination_path', asset_dir)
- task.set_editor_property('destination_name', asset_name)
- task.set_editor_property('replace_existing', False)
- task.set_editor_property('automated', True)
- task.set_editor_property('save', True)
+ if not unreal.EditorAssetLibrary.does_directory_exist(asset_dir):
+ unreal.EditorAssetLibrary.make_directory(asset_dir)
- # set import options here
- # Unreal 4.24 ignores the settings. It works with Unreal 4.26
- options = unreal.AbcImportSettings()
- options.set_editor_property(
- 'import_type', unreal.AlembicImportType.SKELETAL)
+ task = self.get_task(self.fname, asset_dir, asset_name, False)
- task.options = options
- unreal.AssetToolsHelpers.get_asset_tools().import_asset_tasks([task]) # noqa: E501
+ unreal.AssetToolsHelpers.get_asset_tools().import_asset_tasks([task]) # noqa: E501
- # Create Asset Container
- lib.create_avalon_container(
- container=container_name, path=asset_dir)
+ # Create Asset Container
+ unreal_pipeline.create_container(
+ container=container_name, path=asset_dir)
data = {
"schema": "openpype:container-2.0",
- "id": pipeline.AVALON_CONTAINER_ID,
+ "id": AVALON_CONTAINER_ID,
"asset": asset,
"namespace": asset_dir,
"container_name": container_name,
@@ -103,26 +123,11 @@ def load(self, context, name, namespace, data):
def update(self, container, representation):
name = container["asset_name"]
- source_path = api.get_representation_path(representation)
+ source_path = get_representation_path(representation)
destination_path = container["namespace"]
- task = unreal.AssetImportTask()
-
- task.set_editor_property('filename', source_path)
- task.set_editor_property('destination_path', destination_path)
- # strip suffix
- task.set_editor_property('destination_name', name)
- task.set_editor_property('replace_existing', True)
- task.set_editor_property('automated', True)
- task.set_editor_property('save', True)
-
- # set import options here
- # Unreal 4.24 ignores the settings. It works with Unreal 4.26
- options = unreal.AbcImportSettings()
- options.set_editor_property(
- 'import_type', unreal.AlembicImportType.SKELETAL)
+ task = self.get_task(source_path, destination_path, name, True)
- task.options = options
# do import fbx and replace existing data
unreal.AssetToolsHelpers.get_asset_tools().import_asset_tasks([task])
container_path = "{}/{}".format(container["namespace"],
diff --git a/openpype/hosts/unreal/plugins/load/load_alembic_staticmesh.py b/openpype/hosts/unreal/plugins/load/load_alembic_staticmesh.py
index ccec31b8329..50e498dbb0a 100644
--- a/openpype/hosts/unreal/plugins/load/load_alembic_staticmesh.py
+++ b/openpype/hosts/unreal/plugins/load/load_alembic_staticmesh.py
@@ -1,12 +1,17 @@
+# -*- coding: utf-8 -*-
+"""Loader for Static Mesh alembics."""
import os
-from avalon import api, pipeline
-from avalon.unreal import lib
-from avalon.unreal import pipeline as unreal_pipeline
-import unreal
+from openpype.pipeline import (
+ get_representation_path,
+ AVALON_CONTAINER_ID
+)
+from openpype.hosts.unreal.api import plugin
+from openpype.hosts.unreal.api import pipeline as unreal_pipeline
+import unreal # noqa
-class StaticMeshAlembicLoader(api.Loader):
+class StaticMeshAlembicLoader(plugin.Loader):
"""Load Unreal StaticMesh from Alembic"""
families = ["model"]
@@ -19,7 +24,11 @@ def get_task(self, filename, asset_dir, asset_name, replace):
task = unreal.AssetImportTask()
options = unreal.AbcImportSettings()
sm_settings = unreal.AbcStaticMeshSettings()
- conversion_settings = unreal.AbcConversionSettings()
+ conversion_settings = unreal.AbcConversionSettings(
+ preset=unreal.AbcConversionPreset.CUSTOM,
+ flip_u=False, flip_v=False,
+ rotation=[0.0, 0.0, 0.0],
+ scale=[1.0, 1.0, 1.0])
task.set_editor_property('filename', filename)
task.set_editor_property('destination_path', asset_dir)
@@ -35,13 +44,6 @@ def get_task(self, filename, asset_dir, asset_name, replace):
sm_settings.set_editor_property('merge_meshes', True)
- conversion_settings.set_editor_property('flip_u', False)
- conversion_settings.set_editor_property('flip_v', True)
- conversion_settings.set_editor_property(
- 'scale', unreal.Vector(x=100.0, y=100.0, z=100.0))
- conversion_settings.set_editor_property(
- 'rotation', unreal.Vector(x=-90.0, y=0.0, z=180.0))
-
options.static_mesh_settings = sm_settings
options.conversion_settings = conversion_settings
task.options = options
@@ -49,8 +51,7 @@ def get_task(self, filename, asset_dir, asset_name, replace):
return task
def load(self, context, name, namespace, data):
- """
- Load and containerise representation into Content Browser.
+ """Load and containerise representation into Content Browser.
This is two step process. First, import FBX to temporary path and
then call `containerise()` on it - this moves all content to new
@@ -69,36 +70,38 @@ def load(self, context, name, namespace, data):
Returns:
list(str): list of container content
- """
- # Create directory for asset and avalon container
- root = "/Game/Avalon/Assets"
+ """
+ # Create directory for asset and OpenPype container
+ root = "/Game/OpenPype/Assets"
asset = context.get('asset').get('name')
suffix = "_CON"
if asset:
asset_name = "{}_{}".format(asset, name)
else:
asset_name = "{}".format(name)
+ version = context.get('version').get('name')
tools = unreal.AssetToolsHelpers().get_asset_tools()
asset_dir, container_name = tools.create_unique_asset_name(
- "{}/{}/{}".format(root, asset, name), suffix="")
+ f"{root}/{asset}/{name}_v{version:03d}", suffix="")
container_name += suffix
- unreal.EditorAssetLibrary.make_directory(asset_dir)
+ if not unreal.EditorAssetLibrary.does_directory_exist(asset_dir):
+ unreal.EditorAssetLibrary.make_directory(asset_dir)
- task = self.get_task(self.fname, asset_dir, asset_name, False)
+ task = self.get_task(self.fname, asset_dir, asset_name, False)
- unreal.AssetToolsHelpers.get_asset_tools().import_asset_tasks([task]) # noqa: E501
+ unreal.AssetToolsHelpers.get_asset_tools().import_asset_tasks([task]) # noqa: E501
- # Create Asset Container
- lib.create_avalon_container(
- container=container_name, path=asset_dir)
+ # Create Asset Container
+ unreal_pipeline.create_container(
+ container=container_name, path=asset_dir)
data = {
"schema": "openpype:container-2.0",
- "id": pipeline.AVALON_CONTAINER_ID,
+ "id": AVALON_CONTAINER_ID,
"asset": asset,
"namespace": asset_dir,
"container_name": container_name,
@@ -122,7 +125,7 @@ def load(self, context, name, namespace, data):
def update(self, container, representation):
name = container["asset_name"]
- source_path = api.get_representation_path(representation)
+ source_path = get_representation_path(representation)
destination_path = container["namespace"]
task = self.get_task(source_path, destination_path, name, True)
diff --git a/openpype/hosts/unreal/plugins/load/load_animation.py b/openpype/hosts/unreal/plugins/load/load_animation.py
index 481285d603e..1fe0bef4623 100644
--- a/openpype/hosts/unreal/plugins/load/load_animation.py
+++ b/openpype/hosts/unreal/plugins/load/load_animation.py
@@ -1,14 +1,24 @@
+# -*- coding: utf-8 -*-
+"""Load FBX with animations."""
import os
import json
-from avalon import api, pipeline
-from avalon.unreal import lib
-from avalon.unreal import pipeline as unreal_pipeline
import unreal
+from unreal import EditorAssetLibrary
+from unreal import MovieSceneSkeletalAnimationTrack
+from unreal import MovieSceneSkeletalAnimationSection
+from openpype.pipeline.context_tools import get_current_project_asset
+from openpype.pipeline import (
+ get_representation_path,
+ AVALON_CONTAINER_ID
+)
+from openpype.hosts.unreal.api import plugin
+from openpype.hosts.unreal.api import pipeline as unreal_pipeline
-class AnimationFBXLoader(api.Loader):
- """Load Unreal SkeletalMesh from FBX"""
+
+class AnimationFBXLoader(plugin.Loader):
+ """Load Unreal SkeletalMesh from FBX."""
families = ["animation"]
label = "Import FBX Animation"
@@ -16,69 +26,35 @@ class AnimationFBXLoader(api.Loader):
icon = "cube"
color = "orange"
- def load(self, context, name, namespace, options=None):
- """
- Load and containerise representation into Content Browser.
-
- This is two step process. First, import FBX to temporary path and
- then call `containerise()` on it - this moves all content to new
- directory and then it will create AssetContainer there and imprint it
- with metadata. This will mark this path as container.
-
- Args:
- context (dict): application context
- name (str): subset name
- namespace (str): in Unreal this is basically path to container.
- This is not passed here, so namespace is set
- by `containerise()` because only then we know
- real path.
- data (dict): Those would be data to be imprinted. This is not used
- now, data are imprinted by `containerise()`.
-
- Returns:
- list(str): list of container content
- """
-
- # Create directory for asset and avalon container
- root = "/Game/Avalon/Assets"
- asset = context.get('asset').get('name')
- suffix = "_CON"
- if asset:
- asset_name = "{}_{}".format(asset, name)
- else:
- asset_name = "{}".format(name)
-
- tools = unreal.AssetToolsHelpers().get_asset_tools()
- asset_dir, container_name = tools.create_unique_asset_name(
- "{}/{}/{}".format(root, asset, name), suffix="")
-
- container_name += suffix
-
- unreal.EditorAssetLibrary.make_directory(asset_dir)
-
+ def _process(self, asset_dir, asset_name, instance_name):
automated = False
actor = None
task = unreal.AssetImportTask()
task.options = unreal.FbxImportUI()
- libpath = self.fname.replace("fbx", "json")
-
- with open(libpath, "r") as fp:
- data = json.load(fp)
-
- instance_name = data.get("instance_name")
-
if instance_name:
automated = True
- actor_name = 'PersistentLevel.' + instance_name
- actor = unreal.EditorLevelLibrary.get_actor_reference(actor_name)
+ # Old method to get the actor
+ # actor_name = 'PersistentLevel.' + instance_name
+ # actor = unreal.EditorLevelLibrary.get_actor_reference(actor_name)
+ actors = unreal.EditorLevelLibrary.get_all_level_actors()
+ for a in actors:
+ if a.get_class().get_name() != "SkeletalMeshActor":
+ continue
+ if a.get_actor_label() == instance_name:
+ actor = a
+ break
+ if not actor:
+ raise Exception(f"Could not find actor {instance_name}")
skeleton = actor.skeletal_mesh_component.skeletal_mesh.skeleton
task.options.set_editor_property('skeleton', skeleton)
if not actor:
return None
+ asset_doc = get_current_project_asset(fields=["data.fps"])
+
task.set_editor_property('filename', self.fname)
task.set_editor_property('destination_path', asset_dir)
task.set_editor_property('destination_name', asset_name)
@@ -104,46 +80,28 @@ def load(self, context, name, namespace, options=None):
task.options.anim_sequence_import_data.set_editor_property(
'import_meshes_in_bone_hierarchy', False)
task.options.anim_sequence_import_data.set_editor_property(
- 'use_default_sample_rate', True)
+ 'use_default_sample_rate', False)
+ task.options.anim_sequence_import_data.set_editor_property(
+ 'custom_sample_rate', asset_doc.get("data", {}).get("fps"))
task.options.anim_sequence_import_data.set_editor_property(
'import_custom_attribute', True)
task.options.anim_sequence_import_data.set_editor_property(
'import_bone_tracks', True)
task.options.anim_sequence_import_data.set_editor_property(
- 'remove_redundant_keys', True)
+ 'remove_redundant_keys', False)
task.options.anim_sequence_import_data.set_editor_property(
'convert_scene', True)
unreal.AssetToolsHelpers.get_asset_tools().import_asset_tasks([task])
- # Create Asset Container
- lib.create_avalon_container(
- container=container_name, path=asset_dir)
-
- data = {
- "schema": "openpype:container-2.0",
- "id": pipeline.AVALON_CONTAINER_ID,
- "asset": asset,
- "namespace": asset_dir,
- "container_name": container_name,
- "asset_name": asset_name,
- "loader": str(self.__class__.__name__),
- "representation": context["representation"]["_id"],
- "parent": context["representation"]["parent"],
- "family": context["representation"]["context"]["family"]
- }
- unreal_pipeline.imprint(
- "{}/{}".format(asset_dir, container_name), data)
-
- asset_content = unreal.EditorAssetLibrary.list_assets(
+ asset_content = EditorAssetLibrary.list_assets(
asset_dir, recursive=True, include_folder=True
)
animation = None
for a in asset_content:
- unreal.EditorAssetLibrary.save_asset(a)
- imported_asset_data = unreal.EditorAssetLibrary.find_asset_data(a)
+ imported_asset_data = EditorAssetLibrary.find_asset_data(a)
imported_asset = unreal.AssetRegistryHelpers.get_asset(
imported_asset_data)
if imported_asset.__class__ == unreal.AnimSequence:
@@ -157,11 +115,140 @@ def load(self, context, name, namespace, options=None):
actor.skeletal_mesh_component.animation_data.set_editor_property(
'anim_to_play', animation)
- return asset_content
+ return animation
+
+ def load(self, context, name, namespace, options=None):
+ """
+ Load and containerise representation into Content Browser.
+
+ This is two step process. First, import FBX to temporary path and
+ then call `containerise()` on it - this moves all content to new
+ directory and then it will create AssetContainer there and imprint it
+ with metadata. This will mark this path as container.
+
+ Args:
+ context (dict): application context
+ name (str): subset name
+ namespace (str): in Unreal this is basically path to container.
+ This is not passed here, so namespace is set
+ by `containerise()` because only then we know
+ real path.
+ data (dict): Those would be data to be imprinted. This is not used
+ now, data are imprinted by `containerise()`.
+
+ Returns:
+ list(str): list of container content
+ """
+ # Create directory for asset and avalon container
+ hierarchy = context.get('asset').get('data').get('parents')
+ root = "/Game/OpenPype"
+ asset = context.get('asset').get('name')
+ suffix = "_CON"
+ asset_name = f"{asset}_{name}" if asset else f"{name}"
+ tools = unreal.AssetToolsHelpers().get_asset_tools()
+ asset_dir, container_name = tools.create_unique_asset_name(
+ f"{root}/Animations/{asset}/{name}", suffix="")
+
+ ar = unreal.AssetRegistryHelpers.get_asset_registry()
+
+ _filter = unreal.ARFilter(
+ class_names=["World"],
+ package_paths=[f"{root}/{hierarchy[0]}"],
+ recursive_paths=False)
+ levels = ar.get_assets(_filter)
+ master_level = levels[0].get_editor_property('object_path')
+
+ hierarchy_dir = root
+ for h in hierarchy:
+ hierarchy_dir = f"{hierarchy_dir}/{h}"
+ hierarchy_dir = f"{hierarchy_dir}/{asset}"
+
+ _filter = unreal.ARFilter(
+ class_names=["World"],
+ package_paths=[f"{hierarchy_dir}/"],
+ recursive_paths=True)
+ levels = ar.get_assets(_filter)
+ level = levels[0].get_editor_property('object_path')
+
+ unreal.EditorLevelLibrary.save_all_dirty_levels()
+ unreal.EditorLevelLibrary.load_level(level)
+
+ container_name += suffix
+
+ EditorAssetLibrary.make_directory(asset_dir)
+
+ libpath = self.fname.replace("fbx", "json")
+
+ with open(libpath, "r") as fp:
+ data = json.load(fp)
+
+ instance_name = data.get("instance_name")
+
+ animation = self._process(asset_dir, asset_name, instance_name)
+
+ asset_content = EditorAssetLibrary.list_assets(
+ hierarchy_dir, recursive=True, include_folder=False)
+
+ # Get the sequence for the layout, excluding the camera one.
+ sequences = [a for a in asset_content
+ if (EditorAssetLibrary.find_asset_data(a).get_class() ==
+ unreal.LevelSequence.static_class() and
+ "_camera" not in a.split("/")[-1])]
+
+ ar = unreal.AssetRegistryHelpers.get_asset_registry()
+
+ for s in sequences:
+ sequence = ar.get_asset_by_object_path(s).get_asset()
+ possessables = [
+ p for p in sequence.get_possessables()
+ if p.get_display_name() == instance_name]
+
+ for p in possessables:
+ tracks = [
+ t for t in p.get_tracks()
+ if (t.get_class() ==
+ MovieSceneSkeletalAnimationTrack.static_class())]
+
+ for t in tracks:
+ sections = [
+ s for s in t.get_sections()
+ if (s.get_class() ==
+ MovieSceneSkeletalAnimationSection.static_class())]
+
+ for s in sections:
+ s.params.set_editor_property('animation', animation)
+
+ # Create Asset Container
+ unreal_pipeline.create_container(
+ container=container_name, path=asset_dir)
+
+ data = {
+ "schema": "openpype:container-2.0",
+ "id": AVALON_CONTAINER_ID,
+ "asset": asset,
+ "namespace": asset_dir,
+ "container_name": container_name,
+ "asset_name": asset_name,
+ "loader": str(self.__class__.__name__),
+ "representation": context["representation"]["_id"],
+ "parent": context["representation"]["parent"],
+ "family": context["representation"]["context"]["family"]
+ }
+ unreal_pipeline.imprint(f"{asset_dir}/{container_name}", data)
+
+ imported_content = EditorAssetLibrary.list_assets(
+ asset_dir, recursive=True, include_folder=False)
+
+ for a in imported_content:
+ EditorAssetLibrary.save_asset(a)
+
+ unreal.EditorLevelLibrary.save_current_level()
+ unreal.EditorLevelLibrary.load_level(master_level)
def update(self, container, representation):
name = container["asset_name"]
- source_path = api.get_representation_path(representation)
+ source_path = get_representation_path(representation)
+ asset_doc = get_current_project_asset(fields=["data.fps"])
destination_path = container["namespace"]
task = unreal.AssetImportTask()
@@ -173,30 +260,46 @@ def update(self, container, representation):
task.set_editor_property('destination_name', name)
task.set_editor_property('replace_existing', True)
task.set_editor_property('automated', True)
- task.set_editor_property('save', False)
+ task.set_editor_property('save', True)
# set import options here
task.options.set_editor_property(
- 'automated_import_should_detect_type', True)
+ 'automated_import_should_detect_type', False)
task.options.set_editor_property(
- 'original_import_type', unreal.FBXImportType.FBXIT_ANIMATION)
+ 'original_import_type', unreal.FBXImportType.FBXIT_SKELETAL_MESH)
+ task.options.set_editor_property(
+ 'mesh_type_to_import', unreal.FBXImportType.FBXIT_ANIMATION)
task.options.set_editor_property('import_mesh', False)
task.options.set_editor_property('import_animations', True)
+ task.options.set_editor_property('override_full_name', True)
- task.options.skeletal_mesh_import_data.set_editor_property(
- 'import_content_type',
- unreal.FBXImportContentType.FBXICT_SKINNING_WEIGHTS
+ task.options.anim_sequence_import_data.set_editor_property(
+ 'animation_length',
+ unreal.FBXAnimationLengthImportType.FBXALIT_EXPORTED_TIME
)
+ task.options.anim_sequence_import_data.set_editor_property(
+ 'import_meshes_in_bone_hierarchy', False)
+ task.options.anim_sequence_import_data.set_editor_property(
+ 'use_default_sample_rate', False)
+ task.options.anim_sequence_import_data.set_editor_property(
+ 'custom_sample_rate', asset_doc.get("data", {}).get("fps"))
+ task.options.anim_sequence_import_data.set_editor_property(
+ 'import_custom_attribute', True)
+ task.options.anim_sequence_import_data.set_editor_property(
+ 'import_bone_tracks', True)
+ task.options.anim_sequence_import_data.set_editor_property(
+ 'remove_redundant_keys', False)
+ task.options.anim_sequence_import_data.set_editor_property(
+ 'convert_scene', True)
- skeletal_mesh = unreal.EditorAssetLibrary.load_asset(
+ skeletal_mesh = EditorAssetLibrary.load_asset(
container.get('namespace') + "/" + container.get('asset_name'))
skeleton = skeletal_mesh.get_editor_property('skeleton')
task.options.set_editor_property('skeleton', skeleton)
# do import fbx and replace existing data
unreal.AssetToolsHelpers.get_asset_tools().import_asset_tasks([task])
- container_path = "{}/{}".format(container["namespace"],
- container["objectName"])
+ container_path = f'{container["namespace"]}/{container["objectName"]}'
# update metadata
unreal_pipeline.imprint(
container_path,
@@ -205,22 +308,22 @@ def update(self, container, representation):
"parent": str(representation["parent"])
})
- asset_content = unreal.EditorAssetLibrary.list_assets(
+ asset_content = EditorAssetLibrary.list_assets(
destination_path, recursive=True, include_folder=True
)
for a in asset_content:
- unreal.EditorAssetLibrary.save_asset(a)
+ EditorAssetLibrary.save_asset(a)
def remove(self, container):
path = container["namespace"]
parent_path = os.path.dirname(path)
- unreal.EditorAssetLibrary.delete_directory(path)
+ EditorAssetLibrary.delete_directory(path)
- asset_content = unreal.EditorAssetLibrary.list_assets(
- parent_path, recursive=False
+ asset_content = EditorAssetLibrary.list_assets(
+ parent_path, recursive=False, include_folder=True
)
if len(asset_content) == 0:
- unreal.EditorAssetLibrary.delete_directory(parent_path)
+ EditorAssetLibrary.delete_directory(parent_path)
diff --git a/openpype/hosts/unreal/plugins/load/load_camera.py b/openpype/hosts/unreal/plugins/load/load_camera.py
index b2b25eec73b..ca6b0ce7368 100644
--- a/openpype/hosts/unreal/plugins/load/load_camera.py
+++ b/openpype/hosts/unreal/plugins/load/load_camera.py
@@ -1,12 +1,21 @@
-import os
+# -*- coding: utf-8 -*-
+"""Load camera from FBX."""
+from pathlib import Path
-from avalon import api, io, pipeline
-from avalon.unreal import lib
-from avalon.unreal import pipeline as unreal_pipeline
import unreal
-
-
-class CameraLoader(api.Loader):
+from unreal import EditorAssetLibrary
+from unreal import EditorLevelLibrary
+from unreal import EditorLevelUtils
+from openpype.client import get_assets, get_asset_by_name
+from openpype.pipeline import (
+ AVALON_CONTAINER_ID,
+ legacy_io,
+)
+from openpype.hosts.unreal.api import plugin
+from openpype.hosts.unreal.api import pipeline as unreal_pipeline
+
+
+class CameraLoader(plugin.Loader):
"""Load Unreal StaticMesh from FBX"""
families = ["camera"]
@@ -15,6 +24,59 @@ class CameraLoader(api.Loader):
icon = "cube"
color = "orange"
+ def _set_sequence_hierarchy(
+ self, seq_i, seq_j, min_frame_j, max_frame_j
+ ):
+ tracks = seq_i.get_master_tracks()
+ track = None
+ for t in tracks:
+ if t.get_class() == unreal.MovieSceneSubTrack.static_class():
+ track = t
+ break
+ if not track:
+ track = seq_i.add_master_track(unreal.MovieSceneSubTrack)
+
+ subscenes = track.get_sections()
+ subscene = None
+ for s in subscenes:
+ if s.get_editor_property('sub_sequence') == seq_j:
+ subscene = s
+ break
+ if not subscene:
+ subscene = track.add_section()
+ subscene.set_row_index(len(track.get_sections()))
+ subscene.set_editor_property('sub_sequence', seq_j)
+ subscene.set_range(
+ min_frame_j,
+ max_frame_j + 1)
+
+ def _import_camera(
+ self, world, sequence, bindings, import_fbx_settings, import_filename
+ ):
+ ue_version = unreal.SystemLibrary.get_engine_version().split('.')
+ ue_major = int(ue_version[0])
+ ue_minor = int(ue_version[1])
+
+ if ue_major == 4 and ue_minor <= 26:
+ unreal.SequencerTools.import_fbx(
+ world,
+ sequence,
+ bindings,
+ import_fbx_settings,
+ import_filename
+ )
+ elif (ue_major == 4 and ue_minor >= 27) or ue_major == 5:
+ unreal.SequencerTools.import_level_sequence_fbx(
+ world,
+ sequence,
+ bindings,
+ import_fbx_settings,
+ import_filename
+ )
+ else:
+ raise NotImplementedError(
+ f"Unreal version {ue_major} not supported")
+
def load(self, context, name, namespace, data):
"""
Load and containerise representation into Content Browser.
@@ -39,7 +101,13 @@ def load(self, context, name, namespace, data):
"""
# Create directory for asset and avalon container
- root = "/Game/Avalon/Assets"
+ hierarchy = context.get('asset').get('data').get('parents')
+ root = "/Game/OpenPype"
+ hierarchy_dir = root
+ hierarchy_dir_list = []
+ for h in hierarchy:
+ hierarchy_dir = f"{hierarchy_dir}/{h}"
+ hierarchy_dir_list.append(hierarchy_dir)
asset = context.get('asset').get('name')
suffix = "_CON"
if asset:
@@ -49,10 +117,10 @@ def load(self, context, name, namespace, data):
tools = unreal.AssetToolsHelpers().get_asset_tools()
+ # Create a unique name for the camera directory
unique_number = 1
-
- if unreal.EditorAssetLibrary.does_directory_exist(f"{root}/{asset}"):
- asset_content = unreal.EditorAssetLibrary.list_assets(
+ if EditorAssetLibrary.does_directory_exist(f"{hierarchy_dir}/{asset}"):
+ asset_content = EditorAssetLibrary.list_assets(
f"{root}/{asset}", recursive=False, include_folder=True
)
@@ -71,49 +139,160 @@ def load(self, context, name, namespace, data):
unique_number = f_numbers[-1] + 1
asset_dir, container_name = tools.create_unique_asset_name(
- f"{root}/{asset}/{name}_{unique_number:02d}", suffix="")
+ f"{hierarchy_dir}/{asset}/{name}_{unique_number:02d}", suffix="")
+
+ asset_path = Path(asset_dir)
+ asset_path_parent = str(asset_path.parent.as_posix())
container_name += suffix
- unreal.EditorAssetLibrary.make_directory(asset_dir)
+ EditorAssetLibrary.make_directory(asset_dir)
+
+ # Create map for the shot, and create hierarchy of map. If the maps
+ # already exist, we will use them.
+ h_dir = hierarchy_dir_list[0]
+ h_asset = hierarchy[0]
+ master_level = f"{h_dir}/{h_asset}_map.{h_asset}_map"
+ if not EditorAssetLibrary.does_asset_exist(master_level):
+ EditorLevelLibrary.new_level(f"{h_dir}/{h_asset}_map")
+
+ level = f"{asset_path_parent}/{asset}_map.{asset}_map"
+ if not EditorAssetLibrary.does_asset_exist(level):
+ EditorLevelLibrary.new_level(f"{asset_path_parent}/{asset}_map")
+
+ EditorLevelLibrary.load_level(master_level)
+ EditorLevelUtils.add_level_to_world(
+ EditorLevelLibrary.get_editor_world(),
+ level,
+ unreal.LevelStreamingDynamic
+ )
+ EditorLevelLibrary.save_all_dirty_levels()
+ EditorLevelLibrary.load_level(level)
+
+ project_name = legacy_io.active_project()
+ # TODO refactor
+ # - Creationg of hierarchy should be a function in unreal integration
+ # - it's used in multiple loaders but must not be loader's logic
+ # - hard to say what is purpose of the loop
+ # - variables does not match their meaning
+ # - why scene is stored to sequences?
+ # - asset documents vs. elements
+ # - cleanup variable names in whole function
+ # - e.g. 'asset', 'asset_name', 'asset_data', 'asset_doc'
+ # - really inefficient queries of asset documents
+ # - existing asset in scene is considered as "with correct values"
+ # - variable 'elements' is modified during it's loop
+ # Get all the sequences in the hierarchy. It will create them, if
+ # they don't exist.
+ sequences = []
+ frame_ranges = []
+ i = 0
+ for h in hierarchy_dir_list:
+ root_content = EditorAssetLibrary.list_assets(
+ h, recursive=False, include_folder=False)
+
+ existing_sequences = [
+ EditorAssetLibrary.find_asset_data(asset)
+ for asset in root_content
+ if EditorAssetLibrary.find_asset_data(
+ asset).get_class().get_name() == 'LevelSequence'
+ ]
+
+ if not existing_sequences:
+ scene = tools.create_asset(
+ asset_name=hierarchy[i],
+ package_path=h,
+ asset_class=unreal.LevelSequence,
+ factory=unreal.LevelSequenceFactoryNew()
+ )
+
+ asset_data = get_asset_by_name(
+ project_name,
+ h.split('/')[-1],
+ fields=["_id", "data.fps"]
+ )
+
+ start_frames = []
+ end_frames = []
+
+ elements = list(get_assets(
+ project_name,
+ parent_ids=[asset_data["_id"]],
+ fields=["_id", "data.clipIn", "data.clipOut"]
+ ))
+
+ for e in elements:
+ start_frames.append(e.get('data').get('clipIn'))
+ end_frames.append(e.get('data').get('clipOut'))
+
+ elements.extend(get_assets(
+ project_name,
+ parent_ids=[e["_id"]],
+ fields=["_id", "data.clipIn", "data.clipOut"]
+ ))
+
+ min_frame = min(start_frames)
+ max_frame = max(end_frames)
+
+ scene.set_display_rate(
+ unreal.FrameRate(asset_data.get('data').get("fps"), 1.0))
+ scene.set_playback_start(min_frame)
+ scene.set_playback_end(max_frame)
- sequence = tools.create_asset(
- asset_name=asset_name,
+ sequences.append(scene)
+ frame_ranges.append((min_frame, max_frame))
+ else:
+ for e in existing_sequences:
+ sequences.append(e.get_asset())
+ frame_ranges.append((
+ e.get_asset().get_playback_start(),
+ e.get_asset().get_playback_end()))
+
+ i += 1
+
+ EditorAssetLibrary.make_directory(asset_dir)
+
+ cam_seq = tools.create_asset(
+ asset_name=f"{asset}_camera",
package_path=asset_dir,
asset_class=unreal.LevelSequence,
factory=unreal.LevelSequenceFactoryNew()
)
- io_asset = io.Session["AVALON_ASSET"]
- asset_doc = io.find_one({
- "type": "asset",
- "name": io_asset
- })
-
- data = asset_doc.get("data")
-
- if data:
- sequence.set_display_rate(unreal.FrameRate(data.get("fps"), 1.0))
- sequence.set_playback_start(data.get("frameStart"))
- sequence.set_playback_end(data.get("frameEnd"))
+ # Add sequences data to hierarchy
+ for i in range(0, len(sequences) - 1):
+ self._set_sequence_hierarchy(
+ sequences[i], sequences[i + 1],
+ frame_ranges[i + 1][0], frame_ranges[i + 1][1])
+
+ data = get_asset_by_name(project_name, asset)["data"]
+ cam_seq.set_display_rate(
+ unreal.FrameRate(data.get("fps"), 1.0))
+ cam_seq.set_playback_start(0)
+ cam_seq.set_playback_end(data.get('clipOut') - data.get('clipIn') + 1)
+ self._set_sequence_hierarchy(
+ sequences[-1], cam_seq,
+ data.get('clipIn'), data.get('clipOut'))
settings = unreal.MovieSceneUserImportFBXSettings()
settings.set_editor_property('reduce_keys', False)
- unreal.SequencerTools.import_fbx(
- unreal.EditorLevelLibrary.get_editor_world(),
- sequence,
- sequence.get_bindings(),
- settings,
- self.fname
- )
+ if cam_seq:
+ self._import_camera(
+ EditorLevelLibrary.get_editor_world(),
+ cam_seq,
+ cam_seq.get_bindings(),
+ settings,
+ self.fname
+ )
# Create Asset Container
- lib.create_avalon_container(container=container_name, path=asset_dir)
+ unreal_pipeline.create_container(
+ container=container_name, path=asset_dir)
data = {
"schema": "openpype:container-2.0",
- "id": pipeline.AVALON_CONTAINER_ID,
+ "id": AVALON_CONTAINER_ID,
"asset": asset,
"namespace": asset_dir,
"container_name": container_name,
@@ -126,81 +305,258 @@ def load(self, context, name, namespace, data):
unreal_pipeline.imprint(
"{}/{}".format(asset_dir, container_name), data)
- asset_content = unreal.EditorAssetLibrary.list_assets(
+ EditorLevelLibrary.save_all_dirty_levels()
+ EditorLevelLibrary.load_level(master_level)
+
+ asset_content = EditorAssetLibrary.list_assets(
asset_dir, recursive=True, include_folder=True
)
for a in asset_content:
- unreal.EditorAssetLibrary.save_asset(a)
+ EditorAssetLibrary.save_asset(a)
return asset_content
def update(self, container, representation):
- path = container["namespace"]
-
ar = unreal.AssetRegistryHelpers.get_asset_registry()
- tools = unreal.AssetToolsHelpers().get_asset_tools()
- asset_content = unreal.EditorAssetLibrary.list_assets(
- path, recursive=False, include_folder=False
- )
- asset_name = ""
- for a in asset_content:
- asset = ar.get_asset_by_object_path(a)
- if a.endswith("_CON"):
- loaded_asset = unreal.EditorAssetLibrary.load_asset(a)
- unreal.EditorAssetLibrary.set_metadata_tag(
- loaded_asset, "representation", str(representation["_id"])
- )
- unreal.EditorAssetLibrary.set_metadata_tag(
- loaded_asset, "parent", str(representation["parent"])
- )
- asset_name = unreal.EditorAssetLibrary.get_metadata_tag(
- loaded_asset, "asset_name"
- )
- elif asset.asset_class == "LevelSequence":
- unreal.EditorAssetLibrary.delete_asset(a)
+ root = "/Game/OpenPype"
- sequence = tools.create_asset(
- asset_name=asset_name,
- package_path=path,
- asset_class=unreal.LevelSequence,
- factory=unreal.LevelSequenceFactoryNew()
- )
+ asset_dir = container.get('namespace')
+
+ context = representation.get("context")
+
+ hierarchy = context.get('hierarchy').split("/")
+ h_dir = f"{root}/{hierarchy[0]}"
+ h_asset = hierarchy[0]
+ master_level = f"{h_dir}/{h_asset}_map.{h_asset}_map"
+
+ EditorLevelLibrary.save_current_level()
+
+ filter = unreal.ARFilter(
+ class_names=["LevelSequence"],
+ package_paths=[asset_dir],
+ recursive_paths=False)
+ sequences = ar.get_assets(filter)
+ filter = unreal.ARFilter(
+ class_names=["World"],
+ package_paths=[str(Path(asset_dir).parent.as_posix())],
+ recursive_paths=True)
+ maps = ar.get_assets(filter)
+
+ # There should be only one map in the list
+ EditorLevelLibrary.load_level(maps[0].get_full_name())
- io_asset = io.Session["AVALON_ASSET"]
- asset_doc = io.find_one({
- "type": "asset",
- "name": io_asset
- })
+ level_sequence = sequences[0].get_asset()
- data = asset_doc.get("data")
+ display_rate = level_sequence.get_display_rate()
+ playback_start = level_sequence.get_playback_start()
+ playback_end = level_sequence.get_playback_end()
- if data:
- sequence.set_display_rate(unreal.FrameRate(data.get("fps"), 1.0))
- sequence.set_playback_start(data.get("frameStart"))
- sequence.set_playback_end(data.get("frameEnd"))
+ sequence_name = f"{container.get('asset')}_camera"
+
+ # Get the actors in the level sequence.
+ objs = unreal.SequencerTools.get_bound_objects(
+ unreal.EditorLevelLibrary.get_editor_world(),
+ level_sequence,
+ level_sequence.get_bindings(),
+ unreal.SequencerScriptingRange(
+ has_start_value=True,
+ has_end_value=True,
+ inclusive_start=level_sequence.get_playback_start(),
+ exclusive_end=level_sequence.get_playback_end()
+ )
+ )
+
+ # Delete actors from the map
+ for o in objs:
+ if o.bound_objects[0].get_class().get_name() == "CineCameraActor":
+ actor_path = o.bound_objects[0].get_path_name().split(":")[-1]
+ actor = EditorLevelLibrary.get_actor_reference(actor_path)
+ EditorLevelLibrary.destroy_actor(actor)
+
+ # Remove the Level Sequence from the parent.
+ # We need to traverse the hierarchy from the master sequence to find
+ # the level sequence.
+ root = "/Game/OpenPype"
+ namespace = container.get('namespace').replace(f"{root}/", "")
+ ms_asset = namespace.split('/')[0]
+ filter = unreal.ARFilter(
+ class_names=["LevelSequence"],
+ package_paths=[f"{root}/{ms_asset}"],
+ recursive_paths=False)
+ sequences = ar.get_assets(filter)
+ master_sequence = sequences[0].get_asset()
+
+ sequences = [master_sequence]
+
+ parent = None
+ sub_scene = None
+ for s in sequences:
+ tracks = s.get_master_tracks()
+ subscene_track = None
+ for t in tracks:
+ if t.get_class() == unreal.MovieSceneSubTrack.static_class():
+ subscene_track = t
+ break
+ if subscene_track:
+ sections = subscene_track.get_sections()
+ for ss in sections:
+ if ss.get_sequence().get_name() == sequence_name:
+ parent = s
+ sub_scene = ss
+ # subscene_track.remove_section(ss)
+ break
+ sequences.append(ss.get_sequence())
+ # Update subscenes indexes.
+ i = 0
+ for ss in sections:
+ ss.set_row_index(i)
+ i += 1
+
+ if parent:
+ break
+
+ assert parent, "Could not find the parent sequence"
+
+ EditorAssetLibrary.delete_asset(level_sequence.get_path_name())
settings = unreal.MovieSceneUserImportFBXSettings()
settings.set_editor_property('reduce_keys', False)
- unreal.SequencerTools.import_fbx(
- unreal.EditorLevelLibrary.get_editor_world(),
- sequence,
- sequence.get_bindings(),
+ tools = unreal.AssetToolsHelpers().get_asset_tools()
+ new_sequence = tools.create_asset(
+ asset_name=sequence_name,
+ package_path=asset_dir,
+ asset_class=unreal.LevelSequence,
+ factory=unreal.LevelSequenceFactoryNew()
+ )
+
+ new_sequence.set_display_rate(display_rate)
+ new_sequence.set_playback_start(playback_start)
+ new_sequence.set_playback_end(playback_end)
+
+ sub_scene.set_sequence(new_sequence)
+
+ self._import_camera(
+ EditorLevelLibrary.get_editor_world(),
+ new_sequence,
+ new_sequence.get_bindings(),
settings,
str(representation["data"]["path"])
)
- def remove(self, container):
- path = container["namespace"]
- parent_path = os.path.dirname(path)
+ data = {
+ "representation": str(representation["_id"]),
+ "parent": str(representation["parent"])
+ }
+ unreal_pipeline.imprint(
+ "{}/{}".format(asset_dir, container.get('container_name')), data)
+
+ EditorLevelLibrary.save_current_level()
+
+ asset_content = EditorAssetLibrary.list_assets(
+ asset_dir, recursive=True, include_folder=False)
+
+ for a in asset_content:
+ EditorAssetLibrary.save_asset(a)
+
+ EditorLevelLibrary.load_level(master_level)
- unreal.EditorAssetLibrary.delete_directory(path)
+ def remove(self, container):
+ path = Path(container.get("namespace"))
+ parent_path = str(path.parent.as_posix())
- asset_content = unreal.EditorAssetLibrary.list_assets(
+ ar = unreal.AssetRegistryHelpers.get_asset_registry()
+ filter = unreal.ARFilter(
+ class_names=["LevelSequence"],
+ package_paths=[f"{str(path.as_posix())}"],
+ recursive_paths=False)
+ sequences = ar.get_assets(filter)
+
+ if not sequences:
+ raise Exception("Could not find sequence.")
+
+ world = ar.get_asset_by_object_path(
+ EditorLevelLibrary.get_editor_world().get_path_name())
+
+ filter = unreal.ARFilter(
+ class_names=["World"],
+ package_paths=[f"{parent_path}"],
+ recursive_paths=True)
+ maps = ar.get_assets(filter)
+
+ # There should be only one map in the list
+ if not maps:
+ raise Exception("Could not find map.")
+
+ map = maps[0]
+
+ EditorLevelLibrary.save_all_dirty_levels()
+ EditorLevelLibrary.load_level(map.get_full_name())
+
+ # Remove the camera from the level.
+ actors = EditorLevelLibrary.get_all_level_actors()
+
+ for a in actors:
+ if a.__class__ == unreal.CineCameraActor:
+ EditorLevelLibrary.destroy_actor(a)
+
+ EditorLevelLibrary.save_all_dirty_levels()
+ EditorLevelLibrary.load_level(world.get_full_name())
+
+ # There should be only one sequence in the path.
+ sequence_name = sequences[0].asset_name
+
+ # Remove the Level Sequence from the parent.
+ # We need to traverse the hierarchy from the master sequence to find
+ # the level sequence.
+ root = "/Game/OpenPype"
+ namespace = container.get('namespace').replace(f"{root}/", "")
+ ms_asset = namespace.split('/')[0]
+ filter = unreal.ARFilter(
+ class_names=["LevelSequence"],
+ package_paths=[f"{root}/{ms_asset}"],
+ recursive_paths=False)
+ sequences = ar.get_assets(filter)
+ master_sequence = sequences[0].get_asset()
+
+ sequences = [master_sequence]
+
+ parent = None
+ for s in sequences:
+ tracks = s.get_master_tracks()
+ subscene_track = None
+ for t in tracks:
+ if t.get_class() == unreal.MovieSceneSubTrack.static_class():
+ subscene_track = t
+ break
+ if subscene_track:
+ sections = subscene_track.get_sections()
+ for ss in sections:
+ if ss.get_sequence().get_name() == sequence_name:
+ parent = s
+ subscene_track.remove_section(ss)
+ break
+ sequences.append(ss.get_sequence())
+ # Update subscenes indexes.
+ i = 0
+ for ss in sections:
+ ss.set_row_index(i)
+ i += 1
+
+ if parent:
+ break
+
+ assert parent, "Could not find the parent sequence"
+
+ EditorAssetLibrary.delete_directory(str(path.as_posix()))
+
+ # Check if there isn't any more assets in the parent folder, and
+ # delete it if not.
+ asset_content = EditorAssetLibrary.list_assets(
parent_path, recursive=False, include_folder=True
)
if len(asset_content) == 0:
- unreal.EditorAssetLibrary.delete_directory(parent_path)
+ EditorAssetLibrary.delete_directory(parent_path)
diff --git a/openpype/hosts/unreal/plugins/load/load_layout.py b/openpype/hosts/unreal/plugins/load/load_layout.py
new file mode 100644
index 00000000000..926c932a859
--- /dev/null
+++ b/openpype/hosts/unreal/plugins/load/load_layout.py
@@ -0,0 +1,980 @@
+# -*- coding: utf-8 -*-
+"""Loader for layouts."""
+import json
+from pathlib import Path
+
+import unreal
+from unreal import EditorAssetLibrary
+from unreal import EditorLevelLibrary
+from unreal import EditorLevelUtils
+from unreal import AssetToolsHelpers
+from unreal import FBXImportType
+from unreal import MovieSceneLevelVisibilityTrack
+from unreal import MovieSceneSubTrack
+
+from bson.objectid import ObjectId
+
+from openpype.client import get_asset_by_name, get_assets
+from openpype.pipeline import (
+ discover_loader_plugins,
+ loaders_from_representation,
+ load_container,
+ get_representation_path,
+ AVALON_CONTAINER_ID,
+ legacy_io,
+)
+from openpype.pipeline.context_tools import get_current_project_asset
+from openpype.api import get_current_project_settings
+from openpype.hosts.unreal.api import plugin
+from openpype.hosts.unreal.api import pipeline as unreal_pipeline
+
+
+class LayoutLoader(plugin.Loader):
+ """Load Layout from a JSON file"""
+
+ families = ["layout"]
+ representations = ["json"]
+
+ label = "Load Layout"
+ icon = "code-fork"
+ color = "orange"
+ ASSET_ROOT = "/Game/OpenPype"
+
+ def _get_asset_containers(self, path):
+ ar = unreal.AssetRegistryHelpers.get_asset_registry()
+
+ asset_content = EditorAssetLibrary.list_assets(
+ path, recursive=True)
+
+ asset_containers = []
+
+ # Get all the asset containers
+ for a in asset_content:
+ obj = ar.get_asset_by_object_path(a)
+ if obj.get_asset().get_class().get_name() == 'AssetContainer':
+ asset_containers.append(obj)
+
+ return asset_containers
+
+ @staticmethod
+ def _get_fbx_loader(loaders, family):
+ name = ""
+ if family == 'rig':
+ name = "SkeletalMeshFBXLoader"
+ elif family == 'model':
+ name = "StaticMeshFBXLoader"
+ elif family == 'camera':
+ name = "CameraLoader"
+
+ if name == "":
+ return None
+
+ for loader in loaders:
+ if loader.__name__ == name:
+ return loader
+
+ return None
+
+ @staticmethod
+ def _get_abc_loader(loaders, family):
+ name = ""
+ if family == 'rig':
+ name = "SkeletalMeshAlembicLoader"
+ elif family == 'model':
+ name = "StaticMeshAlembicLoader"
+
+ if name == "":
+ return None
+
+ for loader in loaders:
+ if loader.__name__ == name:
+ return loader
+
+ return None
+
+ @staticmethod
+ def _set_sequence_hierarchy(
+ seq_i, seq_j, max_frame_i, min_frame_j, max_frame_j, map_paths
+ ):
+ # Get existing sequencer tracks or create them if they don't exist
+ tracks = seq_i.get_master_tracks()
+ subscene_track = None
+ visibility_track = None
+ for t in tracks:
+ if t.get_class() == unreal.MovieSceneSubTrack.static_class():
+ subscene_track = t
+ if (t.get_class() ==
+ unreal.MovieSceneLevelVisibilityTrack.static_class()):
+ visibility_track = t
+ if not subscene_track:
+ subscene_track = seq_i.add_master_track(unreal.MovieSceneSubTrack)
+ if not visibility_track:
+ visibility_track = seq_i.add_master_track(
+ unreal.MovieSceneLevelVisibilityTrack)
+
+ # Create the sub-scene section
+ subscenes = subscene_track.get_sections()
+ subscene = None
+ for s in subscenes:
+ if s.get_editor_property('sub_sequence') == seq_j:
+ subscene = s
+ break
+ if not subscene:
+ subscene = subscene_track.add_section()
+ subscene.set_row_index(len(subscene_track.get_sections()))
+ subscene.set_editor_property('sub_sequence', seq_j)
+ subscene.set_range(
+ min_frame_j,
+ max_frame_j + 1)
+
+ # Create the visibility section
+ ar = unreal.AssetRegistryHelpers.get_asset_registry()
+ maps = []
+ for m in map_paths:
+ # Unreal requires to load the level to get the map name
+ EditorLevelLibrary.save_all_dirty_levels()
+ EditorLevelLibrary.load_level(m)
+ maps.append(str(ar.get_asset_by_object_path(m).asset_name))
+
+ vis_section = visibility_track.add_section()
+ index = len(visibility_track.get_sections())
+
+ vis_section.set_range(
+ min_frame_j,
+ max_frame_j + 1)
+ vis_section.set_visibility(unreal.LevelVisibility.VISIBLE)
+ vis_section.set_row_index(index)
+ vis_section.set_level_names(maps)
+
+ if min_frame_j > 1:
+ hid_section = visibility_track.add_section()
+ hid_section.set_range(
+ 1,
+ min_frame_j)
+ hid_section.set_visibility(unreal.LevelVisibility.HIDDEN)
+ hid_section.set_row_index(index)
+ hid_section.set_level_names(maps)
+ if max_frame_j < max_frame_i:
+ hid_section = visibility_track.add_section()
+ hid_section.set_range(
+ max_frame_j + 1,
+ max_frame_i + 1)
+ hid_section.set_visibility(unreal.LevelVisibility.HIDDEN)
+ hid_section.set_row_index(index)
+ hid_section.set_level_names(maps)
+
+ def _transform_from_basis(self, transform, basis):
+ """Transform a transform from a basis to a new basis."""
+ # Get the basis matrix
+ basis_matrix = unreal.Matrix(
+ basis[0],
+ basis[1],
+ basis[2],
+ basis[3]
+ )
+ transform_matrix = unreal.Matrix(
+ transform[0],
+ transform[1],
+ transform[2],
+ transform[3]
+ )
+
+ new_transform = (
+ basis_matrix.get_inverse() * transform_matrix * basis_matrix)
+
+ return new_transform.transform()
+
+ def _process_family(
+ self, assets, class_name, transform, basis, sequence, inst_name=None
+ ):
+ ar = unreal.AssetRegistryHelpers.get_asset_registry()
+
+ actors = []
+ bindings = []
+
+ for asset in assets:
+ obj = ar.get_asset_by_object_path(asset).get_asset()
+ if obj.get_class().get_name() == class_name:
+ t = self._transform_from_basis(transform, basis)
+ actor = EditorLevelLibrary.spawn_actor_from_object(
+ obj, t.translation
+ )
+ actor.set_actor_rotation(t.rotation.rotator(), False)
+ actor.set_actor_scale3d(t.scale3d)
+
+ if class_name == 'SkeletalMesh':
+ skm_comp = actor.get_editor_property(
+ 'skeletal_mesh_component')
+ skm_comp.set_bounds_scale(10.0)
+
+ actors.append(actor)
+
+ if sequence:
+ binding = None
+ for p in sequence.get_possessables():
+ if p.get_name() == actor.get_name():
+ binding = p
+ break
+
+ if not binding:
+ binding = sequence.add_possessable(actor)
+
+ bindings.append(binding)
+
+ return actors, bindings
+
+ def _import_animation(
+ self, asset_dir, path, instance_name, skeleton, actors_dict,
+ animation_file, bindings_dict, sequence
+ ):
+ anim_file = Path(animation_file)
+ anim_file_name = anim_file.with_suffix('')
+
+ anim_path = f"{asset_dir}/animations/{anim_file_name}"
+
+ asset_doc = get_current_project_asset()
+ # Import animation
+ task = unreal.AssetImportTask()
+ task.options = unreal.FbxImportUI()
+
+ task.set_editor_property(
+ 'filename', str(path.with_suffix(f".{animation_file}")))
+ task.set_editor_property('destination_path', anim_path)
+ task.set_editor_property(
+ 'destination_name', f"{instance_name}_animation")
+ task.set_editor_property('replace_existing', False)
+ task.set_editor_property('automated', True)
+ task.set_editor_property('save', False)
+
+ # set import options here
+ task.options.set_editor_property(
+ 'automated_import_should_detect_type', False)
+ task.options.set_editor_property(
+ 'original_import_type', FBXImportType.FBXIT_SKELETAL_MESH)
+ task.options.set_editor_property(
+ 'mesh_type_to_import', FBXImportType.FBXIT_ANIMATION)
+ task.options.set_editor_property('import_mesh', False)
+ task.options.set_editor_property('import_animations', True)
+ task.options.set_editor_property('override_full_name', True)
+ task.options.set_editor_property('skeleton', skeleton)
+
+ task.options.anim_sequence_import_data.set_editor_property(
+ 'animation_length',
+ unreal.FBXAnimationLengthImportType.FBXALIT_EXPORTED_TIME
+ )
+ task.options.anim_sequence_import_data.set_editor_property(
+ 'import_meshes_in_bone_hierarchy', False)
+ task.options.anim_sequence_import_data.set_editor_property(
+ 'use_default_sample_rate', False)
+ task.options.anim_sequence_import_data.set_editor_property(
+ 'custom_sample_rate', asset_doc.get("data", {}).get("fps"))
+ task.options.anim_sequence_import_data.set_editor_property(
+ 'import_custom_attribute', True)
+ task.options.anim_sequence_import_data.set_editor_property(
+ 'import_bone_tracks', True)
+ task.options.anim_sequence_import_data.set_editor_property(
+ 'remove_redundant_keys', False)
+ task.options.anim_sequence_import_data.set_editor_property(
+ 'convert_scene', True)
+
+ AssetToolsHelpers.get_asset_tools().import_asset_tasks([task])
+
+ asset_content = unreal.EditorAssetLibrary.list_assets(
+ anim_path, recursive=False, include_folder=False
+ )
+
+ animation = None
+
+ for a in asset_content:
+ unreal.EditorAssetLibrary.save_asset(a)
+ imported_asset_data = unreal.EditorAssetLibrary.find_asset_data(a)
+ imported_asset = unreal.AssetRegistryHelpers.get_asset(
+ imported_asset_data)
+ if imported_asset.__class__ == unreal.AnimSequence:
+ animation = imported_asset
+ break
+
+ if animation:
+ actor = None
+ if actors_dict.get(instance_name):
+ for a in actors_dict.get(instance_name):
+ if a.get_class().get_name() == 'SkeletalMeshActor':
+ actor = a
+ break
+
+ animation.set_editor_property('enable_root_motion', True)
+ actor.skeletal_mesh_component.set_editor_property(
+ 'animation_mode', unreal.AnimationMode.ANIMATION_SINGLE_NODE)
+ actor.skeletal_mesh_component.animation_data.set_editor_property(
+ 'anim_to_play', animation)
+
+ if sequence:
+ # Add animation to the sequencer
+ bindings = bindings_dict.get(instance_name)
+
+ ar = unreal.AssetRegistryHelpers.get_asset_registry()
+
+ for binding in bindings:
+ tracks = binding.get_tracks()
+ track = None
+ track = tracks[0] if tracks else binding.add_track(
+ unreal.MovieSceneSkeletalAnimationTrack)
+
+ sections = track.get_sections()
+ section = None
+ if not sections:
+ section = track.add_section()
+ else:
+ section = sections[0]
+
+ sec_params = section.get_editor_property('params')
+ curr_anim = sec_params.get_editor_property('animation')
+
+ if curr_anim:
+ # Checks if the animation path has a container.
+ # If it does, it means that the animation is
+ # already in the sequencer.
+ anim_path = str(Path(
+ curr_anim.get_path_name()).parent
+ ).replace('\\', '/')
+
+ _filter = unreal.ARFilter(
+ class_names=["AssetContainer"],
+ package_paths=[anim_path],
+ recursive_paths=False)
+ containers = ar.get_assets(_filter)
+
+ if len(containers) > 0:
+ return
+
+ section.set_range(
+ sequence.get_playback_start(),
+ sequence.get_playback_end())
+ sec_params = section.get_editor_property('params')
+ sec_params.set_editor_property('animation', animation)
+
+ @staticmethod
+ def _generate_sequence(h, h_dir):
+ tools = unreal.AssetToolsHelpers().get_asset_tools()
+
+ sequence = tools.create_asset(
+ asset_name=h,
+ package_path=h_dir,
+ asset_class=unreal.LevelSequence,
+ factory=unreal.LevelSequenceFactoryNew()
+ )
+
+ project_name = legacy_io.active_project()
+ asset_data = get_asset_by_name(
+ project_name,
+ h_dir.split('/')[-1],
+ fields=["_id", "data.fps"]
+ )
+
+ start_frames = []
+ end_frames = []
+
+ elements = list(get_assets(
+ project_name,
+ parent_ids=[asset_data["_id"]],
+ fields=["_id", "data.clipIn", "data.clipOut"]
+ ))
+ for e in elements:
+ start_frames.append(e.get('data').get('clipIn'))
+ end_frames.append(e.get('data').get('clipOut'))
+
+ elements.extend(get_assets(
+ project_name,
+ parent_ids=[e["_id"]],
+ fields=["_id", "data.clipIn", "data.clipOut"]
+ ))
+
+ min_frame = min(start_frames)
+ max_frame = max(end_frames)
+
+ sequence.set_display_rate(
+ unreal.FrameRate(asset_data.get('data').get("fps"), 1.0))
+ sequence.set_playback_start(min_frame)
+ sequence.set_playback_end(max_frame)
+
+ tracks = sequence.get_master_tracks()
+ track = None
+ for t in tracks:
+ if (t.get_class() ==
+ unreal.MovieSceneCameraCutTrack.static_class()):
+ track = t
+ break
+ if not track:
+ track = sequence.add_master_track(
+ unreal.MovieSceneCameraCutTrack)
+
+ return sequence, (min_frame, max_frame)
+
+ def _process(self, lib_path, asset_dir, sequence, repr_loaded=None):
+ ar = unreal.AssetRegistryHelpers.get_asset_registry()
+
+ with open(lib_path, "r") as fp:
+ data = json.load(fp)
+
+ all_loaders = discover_loader_plugins()
+
+ if not repr_loaded:
+ repr_loaded = []
+
+ path = Path(lib_path)
+
+ skeleton_dict = {}
+ actors_dict = {}
+ bindings_dict = {}
+
+ loaded_assets = []
+
+ for element in data:
+ representation = None
+ repr_format = None
+ if element.get('representation'):
+ # representation = element.get('representation')
+
+ self.log.info(element.get("version"))
+
+ valid_formats = ['fbx', 'abc']
+
+ repr_data = legacy_io.find_one({
+ "type": "representation",
+ "parent": ObjectId(element.get("version")),
+ "name": {"$in": valid_formats}
+ })
+ repr_format = repr_data.get('name')
+
+ if not repr_data:
+ self.log.error(
+ f"No valid representation found for version "
+ f"{element.get('version')}")
+ continue
+
+ representation = str(repr_data.get('_id'))
+ print(representation)
+ # This is to keep compatibility with old versions of the
+ # json format.
+ elif element.get('reference_fbx'):
+ representation = element.get('reference_fbx')
+ repr_format = 'fbx'
+ elif element.get('reference_abc'):
+ representation = element.get('reference_abc')
+ repr_format = 'abc'
+
+ # If reference is None, this element is skipped, as it cannot be
+ # imported in Unreal
+ if not representation:
+ continue
+
+ instance_name = element.get('instance_name')
+
+ skeleton = None
+
+ if representation not in repr_loaded:
+ repr_loaded.append(representation)
+
+ family = element.get('family')
+ loaders = loaders_from_representation(
+ all_loaders, representation)
+
+ loader = None
+
+ if repr_format == 'fbx':
+ loader = self._get_fbx_loader(loaders, family)
+ elif repr_format == 'abc':
+ loader = self._get_abc_loader(loaders, family)
+
+ if not loader:
+ self.log.error(
+ f"No valid loader found for {representation}")
+ continue
+
+ options = {
+ # "asset_dir": asset_dir
+ }
+
+ assets = load_container(
+ loader,
+ representation,
+ namespace=instance_name,
+ options=options
+ )
+
+ container = None
+
+ for asset in assets:
+ obj = ar.get_asset_by_object_path(asset).get_asset()
+ if obj.get_class().get_name() == 'AssetContainer':
+ container = obj
+ if obj.get_class().get_name() == 'Skeleton':
+ skeleton = obj
+
+ loaded_assets.append(container.get_path_name())
+
+ instances = [
+ item for item in data
+ if ((item.get('version') and
+ item.get('version') == element.get('version')) or
+ item.get('reference_fbx') == representation or
+ item.get('reference_abc') == representation)]
+
+ for instance in instances:
+ # transform = instance.get('transform')
+ transform = instance.get('transform_matrix')
+ basis = instance.get('basis')
+ inst = instance.get('instance_name')
+
+ actors = []
+
+ if family == 'model':
+ actors, _ = self._process_family(
+ assets, 'StaticMesh', transform, basis,
+ sequence, inst
+ )
+ elif family == 'rig':
+ actors, bindings = self._process_family(
+ assets, 'SkeletalMesh', transform, basis,
+ sequence, inst
+ )
+ actors_dict[inst] = actors
+ bindings_dict[inst] = bindings
+
+ if skeleton:
+ skeleton_dict[representation] = skeleton
+ else:
+ skeleton = skeleton_dict.get(representation)
+
+ animation_file = element.get('animation')
+
+ if animation_file and skeleton:
+ self._import_animation(
+ asset_dir, path, instance_name, skeleton, actors_dict,
+ animation_file, bindings_dict, sequence)
+
+ return loaded_assets
+
+ @staticmethod
+ def _remove_family(assets, components, class_name, prop_name):
+ ar = unreal.AssetRegistryHelpers.get_asset_registry()
+
+ objects = []
+ for a in assets:
+ obj = ar.get_asset_by_object_path(a)
+ if obj.get_asset().get_class().get_name() == class_name:
+ objects.append(obj)
+ for obj in objects:
+ for comp in components:
+ if comp.get_editor_property(prop_name) == obj.get_asset():
+ comp.get_owner().destroy_actor()
+
+ def _remove_actors(self, path):
+ asset_containers = self._get_asset_containers(path)
+
+ # Get all the static and skeletal meshes components in the level
+ components = EditorLevelLibrary.get_all_level_actors_components()
+ static_meshes_comp = [
+ c for c in components
+ if c.get_class().get_name() == 'StaticMeshComponent']
+ skel_meshes_comp = [
+ c for c in components
+ if c.get_class().get_name() == 'SkeletalMeshComponent']
+
+ # For all the asset containers, get the static and skeletal meshes.
+ # Then, check the components in the level and destroy the matching
+ # actors.
+ for asset_container in asset_containers:
+ package_path = asset_container.get_editor_property('package_path')
+ family = EditorAssetLibrary.get_metadata_tag(
+ asset_container.get_asset(), 'family')
+ assets = EditorAssetLibrary.list_assets(
+ str(package_path), recursive=False)
+ if family == 'model':
+ self._remove_family(
+ assets, static_meshes_comp, 'StaticMesh', 'static_mesh')
+ elif family == 'rig':
+ self._remove_family(
+ assets, skel_meshes_comp, 'SkeletalMesh', 'skeletal_mesh')
+
+ def load(self, context, name, namespace, options):
+ """Load and containerise representation into Content Browser.
+
+ This is two step process. First, import FBX to temporary path and
+ then call `containerise()` on it - this moves all content to new
+ directory and then it will create AssetContainer there and imprint it
+ with metadata. This will mark this path as container.
+
+ Args:
+ context (dict): application context
+ name (str): subset name
+ namespace (str): in Unreal this is basically path to container.
+ This is not passed here, so namespace is set
+ by `containerise()` because only then we know
+ real path.
+ options (dict): Those would be data to be imprinted. This is not
+ used now, data are imprinted by `containerise()`.
+
+ Returns:
+ list(str): list of container content
+ """
+ data = get_current_project_settings()
+ create_sequences = data["unreal"]["level_sequences_for_layouts"]
+
+ # Create directory for asset and avalon container
+ hierarchy = context.get('asset').get('data').get('parents')
+ root = self.ASSET_ROOT
+ hierarchy_dir = root
+ hierarchy_dir_list = []
+ for h in hierarchy:
+ hierarchy_dir = f"{hierarchy_dir}/{h}"
+ hierarchy_dir_list.append(hierarchy_dir)
+ asset = context.get('asset').get('name')
+ suffix = "_CON"
+ asset_name = f"{asset}_{name}" if asset else name
+
+ tools = unreal.AssetToolsHelpers().get_asset_tools()
+ asset_dir, container_name = tools.create_unique_asset_name(
+ "{}/{}/{}".format(hierarchy_dir, asset, name), suffix="")
+
+ container_name += suffix
+
+ EditorAssetLibrary.make_directory(asset_dir)
+
+ master_level = None
+ shot = None
+ sequences = []
+
+ level = f"{asset_dir}/{asset}_map.{asset}_map"
+ EditorLevelLibrary.new_level(f"{asset_dir}/{asset}_map")
+
+ if create_sequences:
+ # Create map for the shot, and create hierarchy of map. If the
+ # maps already exist, we will use them.
+ if hierarchy:
+ h_dir = hierarchy_dir_list[0]
+ h_asset = hierarchy[0]
+ master_level = f"{h_dir}/{h_asset}_map.{h_asset}_map"
+ if not EditorAssetLibrary.does_asset_exist(master_level):
+ EditorLevelLibrary.new_level(f"{h_dir}/{h_asset}_map")
+
+ if master_level:
+ EditorLevelLibrary.load_level(master_level)
+ EditorLevelUtils.add_level_to_world(
+ EditorLevelLibrary.get_editor_world(),
+ level,
+ unreal.LevelStreamingDynamic
+ )
+ EditorLevelLibrary.save_all_dirty_levels()
+ EditorLevelLibrary.load_level(level)
+
+ # Get all the sequences in the hierarchy. It will create them, if
+ # they don't exist.
+ frame_ranges = []
+ for (h_dir, h) in zip(hierarchy_dir_list, hierarchy):
+ root_content = EditorAssetLibrary.list_assets(
+ h_dir, recursive=False, include_folder=False)
+
+ existing_sequences = [
+ EditorAssetLibrary.find_asset_data(asset)
+ for asset in root_content
+ if EditorAssetLibrary.find_asset_data(
+ asset).get_class().get_name() == 'LevelSequence'
+ ]
+
+ if not existing_sequences:
+ sequence, frame_range = self._generate_sequence(h, h_dir)
+
+ sequences.append(sequence)
+ frame_ranges.append(frame_range)
+ else:
+ for e in existing_sequences:
+ sequences.append(e.get_asset())
+ frame_ranges.append((
+ e.get_asset().get_playback_start(),
+ e.get_asset().get_playback_end()))
+
+ shot = tools.create_asset(
+ asset_name=asset,
+ package_path=asset_dir,
+ asset_class=unreal.LevelSequence,
+ factory=unreal.LevelSequenceFactoryNew()
+ )
+
+ # sequences and frame_ranges have the same length
+ for i in range(0, len(sequences) - 1):
+ self._set_sequence_hierarchy(
+ sequences[i], sequences[i + 1],
+ frame_ranges[i][1],
+ frame_ranges[i + 1][0], frame_ranges[i + 1][1],
+ [level])
+
+ project_name = legacy_io.active_project()
+ data = get_asset_by_name(project_name, asset)["data"]
+ shot.set_display_rate(
+ unreal.FrameRate(data.get("fps"), 1.0))
+ shot.set_playback_start(0)
+ shot.set_playback_end(data.get('clipOut') - data.get('clipIn') + 1)
+ if sequences:
+ self._set_sequence_hierarchy(
+ sequences[-1], shot,
+ frame_ranges[-1][1],
+ data.get('clipIn'), data.get('clipOut'),
+ [level])
+
+ EditorLevelLibrary.load_level(level)
+
+ loaded_assets = self._process(self.fname, asset_dir, shot)
+
+ for s in sequences:
+ EditorAssetLibrary.save_asset(s.get_full_name())
+
+ EditorLevelLibrary.save_current_level()
+
+ # Create Asset Container
+ unreal_pipeline.create_container(
+ container=container_name, path=asset_dir)
+
+ data = {
+ "schema": "openpype:container-2.0",
+ "id": AVALON_CONTAINER_ID,
+ "asset": asset,
+ "namespace": asset_dir,
+ "container_name": container_name,
+ "asset_name": asset_name,
+ "loader": str(self.__class__.__name__),
+ "representation": context["representation"]["_id"],
+ "parent": context["representation"]["parent"],
+ "family": context["representation"]["context"]["family"],
+ "loaded_assets": loaded_assets
+ }
+ unreal_pipeline.imprint(
+ "{}/{}".format(asset_dir, container_name), data)
+
+ asset_content = EditorAssetLibrary.list_assets(
+ asset_dir, recursive=True, include_folder=False)
+
+ for a in asset_content:
+ EditorAssetLibrary.save_asset(a)
+
+ if master_level:
+ EditorLevelLibrary.load_level(master_level)
+
+ return asset_content
+
+ def update(self, container, representation):
+ data = get_current_project_settings()
+ create_sequences = data["unreal"]["level_sequences_for_layouts"]
+
+ ar = unreal.AssetRegistryHelpers.get_asset_registry()
+
+ root = "/Game/OpenPype"
+
+ asset_dir = container.get('namespace')
+ context = representation.get("context")
+
+ sequence = None
+ master_level = None
+
+ if create_sequences:
+ hierarchy = context.get('hierarchy').split("/")
+ h_dir = f"{root}/{hierarchy[0]}"
+ h_asset = hierarchy[0]
+ master_level = f"{h_dir}/{h_asset}_map.{h_asset}_map"
+
+ filter = unreal.ARFilter(
+ class_names=["LevelSequence"],
+ package_paths=[asset_dir],
+ recursive_paths=False)
+ sequences = ar.get_assets(filter)
+ sequence = sequences[0].get_asset()
+
+ prev_level = None
+
+ if not master_level:
+ curr_level = unreal.LevelEditorSubsystem().get_current_level()
+ curr_level_path = curr_level.get_outer().get_path_name()
+ # If the level path does not start with "/Game/", the current
+ # level is a temporary, unsaved level.
+ if curr_level_path.startswith("/Game/"):
+ prev_level = curr_level_path
+
+ # Get layout level
+ filter = unreal.ARFilter(
+ class_names=["World"],
+ package_paths=[asset_dir],
+ recursive_paths=False)
+ levels = ar.get_assets(filter)
+
+ layout_level = levels[0].get_editor_property('object_path')
+
+ EditorLevelLibrary.save_all_dirty_levels()
+ EditorLevelLibrary.load_level(layout_level)
+
+ # Delete all the actors in the level
+ actors = unreal.EditorLevelLibrary.get_all_level_actors()
+ for actor in actors:
+ unreal.EditorLevelLibrary.destroy_actor(actor)
+
+ if create_sequences:
+ EditorLevelLibrary.save_current_level()
+
+ EditorAssetLibrary.delete_directory(f"{asset_dir}/animations/")
+
+ source_path = get_representation_path(representation)
+
+ loaded_assets = self._process(source_path, asset_dir, sequence)
+
+ data = {
+ "representation": str(representation["_id"]),
+ "parent": str(representation["parent"]),
+ "loaded_assets": loaded_assets
+ }
+ unreal_pipeline.imprint(
+ "{}/{}".format(asset_dir, container.get('container_name')), data)
+
+ EditorLevelLibrary.save_current_level()
+
+ asset_content = EditorAssetLibrary.list_assets(
+ asset_dir, recursive=True, include_folder=False)
+
+ for a in asset_content:
+ EditorAssetLibrary.save_asset(a)
+
+ if master_level:
+ EditorLevelLibrary.load_level(master_level)
+ elif prev_level:
+ EditorLevelLibrary.load_level(prev_level)
+
+ def remove(self, container):
+ """
+ Delete the layout. First, check if the assets loaded with the layout
+ are used by other layouts. If not, delete the assets.
+ """
+ data = get_current_project_settings()
+ create_sequences = data["unreal"]["level_sequences_for_layouts"]
+
+ root = "/Game/OpenPype"
+ path = Path(container.get("namespace"))
+
+ containers = unreal_pipeline.ls()
+ layout_containers = [
+ c for c in containers
+ if (c.get('asset_name') != container.get('asset_name') and
+ c.get('family') == "layout")]
+
+ # Check if the assets have been loaded by other layouts, and deletes
+ # them if they haven't.
+ for asset in eval(container.get('loaded_assets')):
+ layouts = [
+ lc for lc in layout_containers
+ if asset in lc.get('loaded_assets')]
+
+ if not layouts:
+ EditorAssetLibrary.delete_directory(str(Path(asset).parent))
+
+ # Delete the parent folder if there aren't any more
+ # layouts in it.
+ asset_content = EditorAssetLibrary.list_assets(
+ str(Path(asset).parent.parent), recursive=False,
+ include_folder=True
+ )
+
+ if len(asset_content) == 0:
+ EditorAssetLibrary.delete_directory(
+ str(Path(asset).parent.parent))
+
+ master_sequence = None
+ master_level = None
+ sequences = []
+
+ if create_sequences:
+ # Remove the Level Sequence from the parent.
+ # We need to traverse the hierarchy from the master sequence to
+ # find the level sequence.
+ namespace = container.get('namespace').replace(f"{root}/", "")
+ ms_asset = namespace.split('/')[0]
+ ar = unreal.AssetRegistryHelpers.get_asset_registry()
+ _filter = unreal.ARFilter(
+ class_names=["LevelSequence"],
+ package_paths=[f"{root}/{ms_asset}"],
+ recursive_paths=False)
+ sequences = ar.get_assets(_filter)
+ master_sequence = sequences[0].get_asset()
+ _filter = unreal.ARFilter(
+ class_names=["World"],
+ package_paths=[f"{root}/{ms_asset}"],
+ recursive_paths=False)
+ levels = ar.get_assets(_filter)
+ master_level = levels[0].get_editor_property('object_path')
+
+ sequences = [master_sequence]
+
+ parent = None
+ for s in sequences:
+ tracks = s.get_master_tracks()
+ subscene_track = None
+ visibility_track = None
+ for t in tracks:
+ if t.get_class() == MovieSceneSubTrack.static_class():
+ subscene_track = t
+ if (t.get_class() ==
+ MovieSceneLevelVisibilityTrack.static_class()):
+ visibility_track = t
+ if subscene_track:
+ sections = subscene_track.get_sections()
+ for ss in sections:
+ if (ss.get_sequence().get_name() ==
+ container.get('asset')):
+ parent = s
+ subscene_track.remove_section(ss)
+ break
+ sequences.append(ss.get_sequence())
+ # Update subscenes indexes.
+ i = 0
+ for ss in sections:
+ ss.set_row_index(i)
+ i += 1
+
+ if visibility_track:
+ sections = visibility_track.get_sections()
+ for ss in sections:
+ if (unreal.Name(f"{container.get('asset')}_map")
+ in ss.get_level_names()):
+ visibility_track.remove_section(ss)
+ # Update visibility sections indexes.
+ i = -1
+ prev_name = []
+ for ss in sections:
+ if prev_name != ss.get_level_names():
+ i += 1
+ ss.set_row_index(i)
+ prev_name = ss.get_level_names()
+ if parent:
+ break
+
+ assert parent, "Could not find the parent sequence"
+
+ # Create a temporary level to delete the layout level.
+ EditorLevelLibrary.save_all_dirty_levels()
+ EditorAssetLibrary.make_directory(f"{root}/tmp")
+ tmp_level = f"{root}/tmp/temp_map"
+ if not EditorAssetLibrary.does_asset_exist(f"{tmp_level}.temp_map"):
+ EditorLevelLibrary.new_level(tmp_level)
+ else:
+ EditorLevelLibrary.load_level(tmp_level)
+
+ # Delete the layout directory.
+ EditorAssetLibrary.delete_directory(str(path))
+
+ if create_sequences:
+ EditorLevelLibrary.load_level(master_level)
+ EditorAssetLibrary.delete_directory(f"{root}/tmp")
+
+ # Delete the parent folder if there aren't any more layouts in it.
+ asset_content = EditorAssetLibrary.list_assets(
+ str(path.parent), recursive=False, include_folder=True
+ )
+
+ if len(asset_content) == 0:
+ EditorAssetLibrary.delete_directory(str(path.parent))
diff --git a/openpype/hosts/unreal/plugins/load/load_rig.py b/openpype/hosts/unreal/plugins/load/load_rig.py
index 7f6e31618ae..227c5c9292a 100644
--- a/openpype/hosts/unreal/plugins/load/load_rig.py
+++ b/openpype/hosts/unreal/plugins/load/load_rig.py
@@ -1,23 +1,27 @@
+# -*- coding: utf-8 -*-
+"""Load Skeletal Meshes form FBX."""
import os
-from avalon import api, pipeline
-from avalon.unreal import lib
-from avalon.unreal import pipeline as unreal_pipeline
-import unreal
+from openpype.pipeline import (
+ get_representation_path,
+ AVALON_CONTAINER_ID
+)
+from openpype.hosts.unreal.api import plugin
+from openpype.hosts.unreal.api import pipeline as unreal_pipeline
+import unreal # noqa
-class SkeletalMeshFBXLoader(api.Loader):
- """Load Unreal SkeletalMesh from FBX"""
+class SkeletalMeshFBXLoader(plugin.Loader):
+ """Load Unreal SkeletalMesh from FBX."""
- families = ["rig"]
+ families = ["rig", "skeletalMesh"]
label = "Import FBX Skeletal Mesh"
representations = ["fbx"]
icon = "cube"
color = "orange"
- def load(self, context, name, namespace, data):
- """
- Load and containerise representation into Content Browser.
+ def load(self, context, name, namespace, options):
+ """Load and containerise representation into Content Browser.
This is two step process. First, import FBX to temporary path and
then call `containerise()` on it - this moves all content to new
@@ -31,73 +35,76 @@ def load(self, context, name, namespace, data):
This is not passed here, so namespace is set
by `containerise()` because only then we know
real path.
- data (dict): Those would be data to be imprinted. This is not used
- now, data are imprinted by `containerise()`.
+ options (dict): Those would be data to be imprinted. This is not
+ used now, data are imprinted by `containerise()`.
Returns:
list(str): list of container content
- """
- # Create directory for asset and avalon container
- root = "/Game/Avalon/Assets"
+ """
+ # Create directory for asset and OpenPype container
+ root = "/Game/OpenPype/Assets"
+ if options and options.get("asset_dir"):
+ root = options["asset_dir"]
asset = context.get('asset').get('name')
suffix = "_CON"
if asset:
asset_name = "{}_{}".format(asset, name)
else:
asset_name = "{}".format(name)
+ version = context.get('version').get('name')
tools = unreal.AssetToolsHelpers().get_asset_tools()
asset_dir, container_name = tools.create_unique_asset_name(
- "{}/{}/{}".format(root, asset, name), suffix="")
+ f"{root}/{asset}/{name}_v{version:03d}", suffix="")
container_name += suffix
- unreal.EditorAssetLibrary.make_directory(asset_dir)
-
- task = unreal.AssetImportTask()
-
- task.set_editor_property('filename', self.fname)
- task.set_editor_property('destination_path', asset_dir)
- task.set_editor_property('destination_name', asset_name)
- task.set_editor_property('replace_existing', False)
- task.set_editor_property('automated', True)
- task.set_editor_property('save', False)
-
- # set import options here
- options = unreal.FbxImportUI()
- options.set_editor_property('import_as_skeletal', True)
- options.set_editor_property('import_animations', False)
- options.set_editor_property('import_mesh', True)
- options.set_editor_property('import_materials', True)
- options.set_editor_property('import_textures', True)
- options.set_editor_property('skeleton', None)
- options.set_editor_property('create_physics_asset', False)
-
- options.set_editor_property('mesh_type_to_import',
- unreal.FBXImportType.FBXIT_SKELETAL_MESH)
-
- options.skeletal_mesh_import_data.set_editor_property(
- 'import_content_type',
- unreal.FBXImportContentType.FBXICT_ALL
- )
- # set to import normals, otherwise Unreal will compute them
- # and it will take a long time, depending on the size of the mesh
- options.skeletal_mesh_import_data.set_editor_property(
- 'normal_import_method',
- unreal.FBXNormalImportMethod.FBXNIM_IMPORT_NORMALS
- )
-
- task.options = options
- unreal.AssetToolsHelpers.get_asset_tools().import_asset_tasks([task]) # noqa: E501
-
- # Create Asset Container
- lib.create_avalon_container(
- container=container_name, path=asset_dir)
+ if not unreal.EditorAssetLibrary.does_directory_exist(asset_dir):
+ unreal.EditorAssetLibrary.make_directory(asset_dir)
+
+ task = unreal.AssetImportTask()
+
+ task.set_editor_property('filename', self.fname)
+ task.set_editor_property('destination_path', asset_dir)
+ task.set_editor_property('destination_name', asset_name)
+ task.set_editor_property('replace_existing', False)
+ task.set_editor_property('automated', True)
+ task.set_editor_property('save', False)
+
+ # set import options here
+ options = unreal.FbxImportUI()
+ options.set_editor_property('import_as_skeletal', True)
+ options.set_editor_property('import_animations', False)
+ options.set_editor_property('import_mesh', True)
+ options.set_editor_property('import_materials', False)
+ options.set_editor_property('import_textures', False)
+ options.set_editor_property('skeleton', None)
+ options.set_editor_property('create_physics_asset', False)
+
+ options.set_editor_property(
+ 'mesh_type_to_import',
+ unreal.FBXImportType.FBXIT_SKELETAL_MESH)
+
+ options.skeletal_mesh_import_data.set_editor_property(
+ 'import_content_type',
+ unreal.FBXImportContentType.FBXICT_ALL)
+ # set to import normals, otherwise Unreal will compute them
+ # and it will take a long time, depending on the size of the mesh
+ options.skeletal_mesh_import_data.set_editor_property(
+ 'normal_import_method',
+ unreal.FBXNormalImportMethod.FBXNIM_IMPORT_NORMALS)
+
+ task.options = options
+ unreal.AssetToolsHelpers.get_asset_tools().import_asset_tasks([task]) # noqa: E501
+
+ # Create Asset Container
+ unreal_pipeline.create_container(
+ container=container_name, path=asset_dir)
data = {
"schema": "openpype:container-2.0",
- "id": pipeline.AVALON_CONTAINER_ID,
+ "id": AVALON_CONTAINER_ID,
"asset": asset,
"namespace": asset_dir,
"container_name": container_name,
@@ -121,7 +128,7 @@ def load(self, context, name, namespace, data):
def update(self, container, representation):
name = container["asset_name"]
- source_path = api.get_representation_path(representation)
+ source_path = get_representation_path(representation)
destination_path = container["namespace"]
task = unreal.AssetImportTask()
diff --git a/openpype/hosts/unreal/plugins/load/load_staticmeshfbx.py b/openpype/hosts/unreal/plugins/load/load_staticmeshfbx.py
index d25f84ea690..351c6860951 100644
--- a/openpype/hosts/unreal/plugins/load/load_staticmeshfbx.py
+++ b/openpype/hosts/unreal/plugins/load/load_staticmeshfbx.py
@@ -1,21 +1,27 @@
+# -*- coding: utf-8 -*-
+"""Load Static meshes form FBX."""
import os
-from avalon import api, pipeline
-from avalon.unreal import lib
-from avalon.unreal import pipeline as unreal_pipeline
-import unreal
+from openpype.pipeline import (
+ get_representation_path,
+ AVALON_CONTAINER_ID
+)
+from openpype.hosts.unreal.api import plugin
+from openpype.hosts.unreal.api import pipeline as unreal_pipeline
+import unreal # noqa
-class StaticMeshFBXLoader(api.Loader):
- """Load Unreal StaticMesh from FBX"""
+class StaticMeshFBXLoader(plugin.Loader):
+ """Load Unreal StaticMesh from FBX."""
- families = ["model", "unrealStaticMesh"]
+ families = ["model", "staticMesh"]
label = "Import FBX Static Mesh"
representations = ["fbx"]
icon = "cube"
color = "orange"
- def get_task(self, filename, asset_dir, asset_name, replace):
+ @staticmethod
+ def get_task(filename, asset_dir, asset_name, replace):
task = unreal.AssetImportTask()
options = unreal.FbxImportUI()
import_data = unreal.FbxStaticMeshImportData()
@@ -40,9 +46,8 @@ def get_task(self, filename, asset_dir, asset_name, replace):
return task
- def load(self, context, name, namespace, data):
- """
- Load and containerise representation into Content Browser.
+ def load(self, context, name, namespace, options):
+ """Load and containerise representation into Content Browser.
This is two step process. First, import FBX to temporary path and
then call `containerise()` on it - this moves all content to new
@@ -56,15 +61,17 @@ def load(self, context, name, namespace, data):
This is not passed here, so namespace is set
by `containerise()` because only then we know
real path.
- data (dict): Those would be data to be imprinted. This is not used
- now, data are imprinted by `containerise()`.
+ options (dict): Those would be data to be imprinted. This is not
+ used now, data are imprinted by `containerise()`.
Returns:
list(str): list of container content
"""
- # Create directory for asset and avalon container
- root = "/Game/Avalon/Assets"
+ # Create directory for asset and OpenPype container
+ root = "/Game/OpenPype/Assets"
+ if options and options.get("asset_dir"):
+ root = options["asset_dir"]
asset = context.get('asset').get('name')
suffix = "_CON"
if asset:
@@ -85,12 +92,12 @@ def load(self, context, name, namespace, data):
unreal.AssetToolsHelpers.get_asset_tools().import_asset_tasks([task]) # noqa: E501
# Create Asset Container
- lib.create_avalon_container(
+ unreal_pipeline.create_container(
container=container_name, path=asset_dir)
data = {
"schema": "openpype:container-2.0",
- "id": pipeline.AVALON_CONTAINER_ID,
+ "id": AVALON_CONTAINER_ID,
"asset": asset,
"namespace": asset_dir,
"container_name": container_name,
@@ -114,7 +121,7 @@ def load(self, context, name, namespace, data):
def update(self, container, representation):
name = container["asset_name"]
- source_path = api.get_representation_path(representation)
+ source_path = get_representation_path(representation)
destination_path = container["namespace"]
task = self.get_task(source_path, destination_path, name, True)
diff --git a/openpype/hosts/unreal/plugins/publish/collect_current_file.py b/openpype/hosts/unreal/plugins/publish/collect_current_file.py
index 4e828933bb8..acd4c5c8d26 100644
--- a/openpype/hosts/unreal/plugins/publish/collect_current_file.py
+++ b/openpype/hosts/unreal/plugins/publish/collect_current_file.py
@@ -1,17 +1,18 @@
-import unreal
-
+# -*- coding: utf-8 -*-
+"""Collect current project path."""
+import unreal # noqa
import pyblish.api
class CollectUnrealCurrentFile(pyblish.api.ContextPlugin):
- """Inject the current working file into context"""
+ """Inject the current working file into context."""
order = pyblish.api.CollectorOrder - 0.5
label = "Unreal Current File"
hosts = ['unreal']
def process(self, context):
- """Inject the current working file"""
+ """Inject the current working file."""
current_file = unreal.Paths.get_project_file_path()
context.data['currentFile'] = current_file
diff --git a/openpype/hosts/unreal/plugins/publish/collect_instances.py b/openpype/hosts/unreal/plugins/publish/collect_instances.py
index 62676f9938d..2f604cb322c 100644
--- a/openpype/hosts/unreal/plugins/publish/collect_instances.py
+++ b/openpype/hosts/unreal/plugins/publish/collect_instances.py
@@ -1,12 +1,14 @@
+# -*- coding: utf-8 -*-
+"""Collect publishable instances in Unreal."""
import ast
-import unreal
+import unreal # noqa
import pyblish.api
class CollectInstances(pyblish.api.ContextPlugin):
- """Gather instances by AvalonPublishInstance class
+ """Gather instances by OpenPypePublishInstance class
- This collector finds all paths containing `AvalonPublishInstance` class
+ This collector finds all paths containing `OpenPypePublishInstance` class
asset
Identifier:
@@ -15,14 +17,14 @@ class CollectInstances(pyblish.api.ContextPlugin):
"""
label = "Collect Instances"
- order = pyblish.api.CollectorOrder
+ order = pyblish.api.CollectorOrder - 0.1
hosts = ["unreal"]
def process(self, context):
ar = unreal.AssetRegistryHelpers.get_asset_registry()
instance_containers = ar.get_assets_by_class(
- "AvalonPublishInstance", True)
+ "OpenPypePublishInstance", True)
for container_data in instance_containers:
asset = container_data.get_asset()
diff --git a/openpype/hosts/unreal/plugins/publish/collect_remove_marked.py b/openpype/hosts/unreal/plugins/publish/collect_remove_marked.py
new file mode 100644
index 00000000000..69e69f66301
--- /dev/null
+++ b/openpype/hosts/unreal/plugins/publish/collect_remove_marked.py
@@ -0,0 +1,24 @@
+import pyblish.api
+
+
+class CollectRemoveMarked(pyblish.api.ContextPlugin):
+ """Remove marked data
+
+ Remove instances that have 'remove' in their instance.data
+
+ """
+
+ order = pyblish.api.CollectorOrder + 0.499
+ label = 'Remove Marked Instances'
+
+ def process(self, context):
+
+ self.log.debug(context)
+ # make ftrack publishable
+ instances_to_remove = []
+ for instance in context:
+ if instance.data.get('remove'):
+ instances_to_remove.append(instance)
+
+ for instance in instances_to_remove:
+ context.remove(instance)
diff --git a/openpype/hosts/unreal/plugins/publish/collect_render_instances.py b/openpype/hosts/unreal/plugins/publish/collect_render_instances.py
new file mode 100644
index 00000000000..cb28f4bf60f
--- /dev/null
+++ b/openpype/hosts/unreal/plugins/publish/collect_render_instances.py
@@ -0,0 +1,112 @@
+import os
+from pathlib import Path
+
+import unreal
+
+from openpype.pipeline import Anatomy
+from openpype.hosts.unreal.api import pipeline
+import pyblish.api
+
+
+class CollectRenderInstances(pyblish.api.InstancePlugin):
+ """ This collector will try to find all the rendered frames.
+
+ """
+ order = pyblish.api.CollectorOrder
+ hosts = ["unreal"]
+ families = ["render"]
+ label = "Collect Render Instances"
+
+ def process(self, instance):
+ self.log.debug("Preparing Rendering Instances")
+
+ context = instance.context
+
+ data = instance.data
+ data['remove'] = True
+
+ ar = unreal.AssetRegistryHelpers.get_asset_registry()
+
+ sequence = ar.get_asset_by_object_path(
+ data.get('sequence')).get_asset()
+
+ sequences = [{
+ "sequence": sequence,
+ "output": data.get('output'),
+ "frame_range": (
+ data.get('frameStart'), data.get('frameEnd'))
+ }]
+
+ for s in sequences:
+ self.log.debug(f"Processing: {s.get('sequence').get_name()}")
+ subscenes = pipeline.get_subsequences(s.get('sequence'))
+
+ if subscenes:
+ for ss in subscenes:
+ sequences.append({
+ "sequence": ss.get_sequence(),
+ "output": (f"{s.get('output')}/"
+ f"{ss.get_sequence().get_name()}"),
+ "frame_range": (
+ ss.get_start_frame(), ss.get_end_frame() - 1)
+ })
+ else:
+ # Avoid creating instances for camera sequences
+ if "_camera" not in s.get('sequence').get_name():
+ seq = s.get('sequence')
+ seq_name = seq.get_name()
+
+ new_instance = context.create_instance(
+ f"{data.get('subset')}_"
+ f"{seq_name}")
+ new_instance[:] = seq_name
+
+ new_data = new_instance.data
+
+ new_data["asset"] = seq_name
+ new_data["setMembers"] = seq_name
+ new_data["family"] = "render"
+ new_data["families"] = ["render", "review"]
+ new_data["parent"] = data.get("parent")
+ new_data["subset"] = f"{data.get('subset')}_{seq_name}"
+ new_data["level"] = data.get("level")
+ new_data["output"] = s.get('output')
+ new_data["fps"] = seq.get_display_rate().numerator
+ new_data["frameStart"] = s.get('frame_range')[0]
+ new_data["frameEnd"] = s.get('frame_range')[1]
+ new_data["sequence"] = seq.get_path_name()
+ new_data["master_sequence"] = data["master_sequence"]
+ new_data["master_level"] = data["master_level"]
+
+ self.log.debug(f"new instance data: {new_data}")
+
+ try:
+ project = os.environ.get("AVALON_PROJECT")
+ anatomy = Anatomy(project)
+ root = anatomy.roots['renders']
+ except Exception:
+ raise Exception(
+ "Could not find render root in anatomy settings.")
+
+ render_dir = f"{root}/{project}/{s.get('output')}"
+ render_path = Path(render_dir)
+
+ frames = []
+
+ for x in render_path.iterdir():
+ if x.is_file() and x.suffix == '.png':
+ frames.append(str(x.name))
+
+ if "representations" not in new_instance.data:
+ new_instance.data["representations"] = []
+
+ repr = {
+ 'frameStart': s.get('frame_range')[0],
+ 'frameEnd': s.get('frame_range')[1],
+ 'name': 'png',
+ 'ext': 'png',
+ 'files': frames,
+ 'stagingDir': render_dir,
+ 'tags': ['review']
+ }
+ new_instance.data["representations"].append(repr)
diff --git a/openpype/hosts/unreal/plugins/publish/extract_camera.py b/openpype/hosts/unreal/plugins/publish/extract_camera.py
index 10862fc0ef2..ce538245630 100644
--- a/openpype/hosts/unreal/plugins/publish/extract_camera.py
+++ b/openpype/hosts/unreal/plugins/publish/extract_camera.py
@@ -1,3 +1,5 @@
+# -*- coding: utf-8 -*-
+"""Extract camera from Unreal."""
import os
import unreal
@@ -17,7 +19,7 @@ class ExtractCamera(openpype.api.Extractor):
def process(self, instance):
# Define extract output file path
- stagingdir = self.staging_dir(instance)
+ staging_dir = self.staging_dir(instance)
fbx_filename = "{}.fbx".format(instance.name)
# Perform extraction
@@ -38,7 +40,7 @@ def process(self, instance):
sequence,
sequence.get_bindings(),
unreal.FbxExportOption(),
- os.path.join(stagingdir, fbx_filename)
+ os.path.join(staging_dir, fbx_filename)
)
break
@@ -49,6 +51,6 @@ def process(self, instance):
'name': 'fbx',
'ext': 'fbx',
'files': fbx_filename,
- "stagingDir": stagingdir,
+ "stagingDir": staging_dir,
}
instance.data["representations"].append(fbx_representation)
diff --git a/openpype/hosts/unreal/plugins/publish/extract_layout.py b/openpype/hosts/unreal/plugins/publish/extract_layout.py
index a47187cf47b..8924df36a7f 100644
--- a/openpype/hosts/unreal/plugins/publish/extract_layout.py
+++ b/openpype/hosts/unreal/plugins/publish/extract_layout.py
@@ -1,13 +1,17 @@
+# -*- coding: utf-8 -*-
import os
import json
import math
+from bson.objectid import ObjectId
+
import unreal
from unreal import EditorLevelLibrary as ell
from unreal import EditorAssetLibrary as eal
+from openpype.client import get_representation_by_name
import openpype.api
-from avalon import io
+from openpype.pipeline import legacy_io
class ExtractLayout(openpype.api.Extractor):
@@ -20,7 +24,7 @@ class ExtractLayout(openpype.api.Extractor):
def process(self, instance):
# Define extract output file path
- stagingdir = self.staging_dir(instance)
+ staging_dir = self.staging_dir(instance)
# Perform extraction
self.log.info("Performing extraction..")
@@ -31,6 +35,7 @@ def process(self, instance):
"Wrong level loaded"
json_data = []
+ project_name = legacy_io.active_project()
for member in instance[:]:
actor = ell.get_actor_reference(member)
@@ -54,17 +59,13 @@ def process(self, instance):
self.log.error("AssetContainer not found.")
return
- parent = eal.get_metadata_tag(asset_container, "parent")
+ parent_id = eal.get_metadata_tag(asset_container, "parent")
family = eal.get_metadata_tag(asset_container, "family")
- self.log.info("Parent: {}".format(parent))
- blend = io.find_one(
- {
- "type": "representation",
- "parent": io.ObjectId(parent),
- "name": "blend"
- },
- projection={"_id": True})
+ self.log.info("Parent: {}".format(parent_id))
+ blend = get_representation_by_name(
+ project_name, "blend", parent_id, fields=["_id"]
+ )
blend_id = blend["_id"]
json_element = {}
@@ -96,7 +97,7 @@ def process(self, instance):
json_data.append(json_element)
json_filename = "{}.json".format(instance.name)
- json_path = os.path.join(stagingdir, json_filename)
+ json_path = os.path.join(staging_dir, json_filename)
with open(json_path, "w+") as file:
json.dump(json_data, fp=file, indent=2)
@@ -108,6 +109,6 @@ def process(self, instance):
'name': 'json',
'ext': 'json',
'files': json_filename,
- "stagingDir": stagingdir,
+ "stagingDir": staging_dir,
}
instance.data["representations"].append(json_representation)
diff --git a/openpype/hosts/unreal/plugins/publish/extract_look.py b/openpype/hosts/unreal/plugins/publish/extract_look.py
index 0f1539a7d57..ea399494173 100644
--- a/openpype/hosts/unreal/plugins/publish/extract_look.py
+++ b/openpype/hosts/unreal/plugins/publish/extract_look.py
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
import json
import os
@@ -17,7 +18,7 @@ class ExtractLook(openpype.api.Extractor):
def process(self, instance):
# Define extract output file path
- stagingdir = self.staging_dir(instance)
+ staging_dir = self.staging_dir(instance)
resources_dir = instance.data["resourcesDir"]
ar = unreal.AssetRegistryHelpers.get_asset_registry()
@@ -57,7 +58,7 @@ def process(self, instance):
tga_export_task.set_editor_property('automated', True)
tga_export_task.set_editor_property('object', texture)
tga_export_task.set_editor_property(
- 'filename', f"{stagingdir}/{tga_filename}")
+ 'filename', f"{staging_dir}/{tga_filename}")
tga_export_task.set_editor_property('prompt', False)
tga_export_task.set_editor_property('selected', False)
@@ -66,7 +67,7 @@ def process(self, instance):
json_element['tga_filename'] = tga_filename
transfers.append((
- f"{stagingdir}/{tga_filename}",
+ f"{staging_dir}/{tga_filename}",
f"{resources_dir}/{tga_filename}"))
fbx_filename = f"{instance.name}_{name}.fbx"
@@ -84,7 +85,7 @@ def process(self, instance):
task.set_editor_property('automated', True)
task.set_editor_property('object', object)
task.set_editor_property(
- 'filename', f"{stagingdir}/{fbx_filename}")
+ 'filename', f"{staging_dir}/{fbx_filename}")
task.set_editor_property('prompt', False)
task.set_editor_property('selected', False)
@@ -93,13 +94,13 @@ def process(self, instance):
json_element['fbx_filename'] = fbx_filename
transfers.append((
- f"{stagingdir}/{fbx_filename}",
+ f"{staging_dir}/{fbx_filename}",
f"{resources_dir}/{fbx_filename}"))
json_data.append(json_element)
json_filename = f"{instance.name}.json"
- json_path = os.path.join(stagingdir, json_filename)
+ json_path = os.path.join(staging_dir, json_filename)
with open(json_path, "w+") as file:
json.dump(json_data, fp=file, indent=2)
@@ -113,7 +114,7 @@ def process(self, instance):
'name': 'json',
'ext': 'json',
'files': json_filename,
- "stagingDir": stagingdir,
+ "stagingDir": staging_dir,
}
instance.data["representations"].append(json_representation)
diff --git a/openpype/hosts/unreal/plugins/publish/extract_render.py b/openpype/hosts/unreal/plugins/publish/extract_render.py
new file mode 100644
index 00000000000..37fe7e916fd
--- /dev/null
+++ b/openpype/hosts/unreal/plugins/publish/extract_render.py
@@ -0,0 +1,48 @@
+from pathlib import Path
+
+import unreal
+
+import openpype.api
+
+
+class ExtractRender(openpype.api.Extractor):
+ """Extract render."""
+
+ label = "Extract Render"
+ hosts = ["unreal"]
+ families = ["render"]
+ optional = True
+
+ def process(self, instance):
+ # Define extract output file path
+ stagingdir = self.staging_dir(instance)
+
+ # Perform extraction
+ self.log.info("Performing extraction..")
+
+ # Get the render output directory
+ project_dir = unreal.Paths.project_dir()
+ render_dir = (f"{project_dir}/Saved/MovieRenders/"
+ f"{instance.data['subset']}")
+
+ assert unreal.Paths.directory_exists(render_dir), \
+ "Render directory does not exist"
+
+ render_path = Path(render_dir)
+
+ frames = []
+
+ for x in render_path.iterdir():
+ if x.is_file() and x.suffix == '.png':
+ frames.append(str(x))
+
+ if "representations" not in instance.data:
+ instance.data["representations"] = []
+
+ render_representation = {
+ 'name': 'png',
+ 'ext': 'png',
+ 'files': frames,
+ "stagingDir": stagingdir,
+ }
+ instance.data["representations"].append(render_representation)
diff --git a/openpype/hosts/unreal/plugins/publish/validate_sequence_frames.py b/openpype/hosts/unreal/plugins/publish/validate_sequence_frames.py
new file mode 100644
index 00000000000..87f1338ee85
--- /dev/null
+++ b/openpype/hosts/unreal/plugins/publish/validate_sequence_frames.py
@@ -0,0 +1,41 @@
+import clique
+
+import pyblish.api
+
+
+class ValidateSequenceFrames(pyblish.api.InstancePlugin):
+ """Ensure the sequence of frames is complete
+
+ The files found in the folder are checked against the frameStart and
+ frameEnd of the instance. If the first or last file is not
+ corresponding with the first or last frame it is flagged as invalid.
+ """
+
+ order = pyblish.api.ValidatorOrder
+ label = "Validate Sequence Frames"
+ families = ["render"]
+ hosts = ["unreal"]
+ optional = True
+
+ def process(self, instance):
+ representations = instance.data.get("representations")
+ for repr in representations:
+ patterns = [clique.PATTERNS["frames"]]
+ collections, remainder = clique.assemble(
+ repr["files"], minimum_items=1, patterns=patterns)
+
+ assert not remainder, "Must not have remainder"
+ assert len(collections) == 1, "Must detect single collection"
+ collection = collections[0]
+ frames = list(collection.indexes)
+
+ current_range = (frames[0], frames[-1])
+ required_range = (instance.data["frameStart"],
+ instance.data["frameEnd"])
+
+ if current_range != required_range:
+ raise ValueError(f"Invalid frame range: {current_range} - "
+ f"expected: {required_range}")
+
+ missing = collection.holes().indexes
+ assert not missing, "Missing frames: %s" % (missing,)
diff --git a/openpype/hosts/webpublisher/__init__.py b/openpype/hosts/webpublisher/__init__.py
index e69de29bb2d..4e918c5d7d6 100644
--- a/openpype/hosts/webpublisher/__init__.py
+++ b/openpype/hosts/webpublisher/__init__.py
@@ -0,0 +1,10 @@
+from .addon import (
+ WebpublisherAddon,
+ WEBPUBLISHER_ROOT_DIR,
+)
+
+
+__all__ = (
+ "WebpublisherAddon",
+ "WEBPUBLISHER_ROOT_DIR",
+)
diff --git a/openpype/hosts/webpublisher/addon.py b/openpype/hosts/webpublisher/addon.py
new file mode 100644
index 00000000000..a64d74e62ba
--- /dev/null
+++ b/openpype/hosts/webpublisher/addon.py
@@ -0,0 +1,106 @@
+import os
+
+import click
+
+from openpype.modules import OpenPypeModule
+from openpype.modules.interfaces import IHostAddon
+
+WEBPUBLISHER_ROOT_DIR = os.path.dirname(os.path.abspath(__file__))
+
+
+class WebpublisherAddon(OpenPypeModule, IHostAddon):
+ name = "webpublisher"
+ host_name = "webpublisher"
+
+ def initialize(self, module_settings):
+ self.enabled = True
+
+ def headless_publish(self, log, close_plugin_name=None, is_test=False):
+ """Runs publish in a opened host with a context.
+
+ Close Python process at the end.
+ """
+
+ from openpype.pipeline.publish.lib import remote_publish
+ from .lib import get_webpublish_conn, publish_and_log
+
+ if is_test:
+ remote_publish(log, close_plugin_name)
+ return
+
+ dbcon = get_webpublish_conn()
+ _id = os.environ.get("BATCH_LOG_ID")
+ if not _id:
+ log.warning("Unable to store log records, "
+ "batch will be unfinished!")
+ return
+
+ publish_and_log(
+ dbcon, _id, log, close_plugin_name=close_plugin_name
+ )
+
+ def cli(self, click_group):
+ click_group.add_command(cli_main)
+
+
+@click.group(
+ WebpublisherAddon.name,
+ help="Webpublisher related commands.")
+def cli_main():
+ pass
+
+
+@cli_main.command()
+@click.argument("path")
+@click.option("-u", "--user", help="User email address")
+@click.option("-p", "--project", help="Project")
+@click.option("-t", "--targets", help="Targets", default=None,
+ multiple=True)
+def publish(project, path, user=None, targets=None):
+ """Start publishing (Inner command).
+
+ Publish collects json from paths provided as an argument.
+ More than one path is allowed.
+ """
+
+ from .publish_functions import cli_publish
+
+ cli_publish(project, path, user, targets)
+
+
+@cli_main.command()
+@click.argument("path")
+@click.option("-p", "--project", help="Project")
+@click.option("-h", "--host", help="Host")
+@click.option("-u", "--user", help="User email address")
+@click.option("-t", "--targets", help="Targets", default=None,
+ multiple=True)
+def publishfromapp(project, path, host, user=None, targets=None):
+ """Start publishing through application (Inner command).
+
+ Publish collects json from paths provided as an argument.
+ More than one path is allowed.
+ """
+
+ from .publish_functions import cli_publish_from_app
+
+ cli_publish_from_app(project, path, host, user, targets)
+
+
+@cli_main.command()
+@click.option("-e", "--executable", help="Executable")
+@click.option("-u", "--upload_dir", help="Upload dir")
+@click.option("-h", "--host", help="Host", default=None)
+@click.option("-p", "--port", help="Port", default=None)
+def webserver(executable, upload_dir, host=None, port=None):
+ """Start service for communication with Webpublish Front end.
+
+ OP must be congigured on a machine, eg. OPENPYPE_MONGO filled AND
+ FTRACK_BOT_API_KEY provided with api key from Ftrack.
+
+ Expect "pype.club" user created on Ftrack.
+ """
+
+ from .webserver_service import run_webserver
+
+ run_webserver(executable, upload_dir, host, port)
diff --git a/openpype/hosts/webpublisher/api/__init__.py b/openpype/hosts/webpublisher/api/__init__.py
index e40d46d6627..afea838e2c7 100644
--- a/openpype/hosts/webpublisher/api/__init__.py
+++ b/openpype/hosts/webpublisher/api/__init__.py
@@ -1,43 +1,23 @@
import os
import logging
-from avalon import api as avalon
-from avalon import io
-from pyblish import api as pyblish
-import openpype.hosts.webpublisher
+import pyblish.api
-log = logging.getLogger("openpype.hosts.webpublisher")
-
-HOST_DIR = os.path.dirname(os.path.abspath(
- openpype.hosts.webpublisher.__file__))
-PLUGINS_DIR = os.path.join(HOST_DIR, "plugins")
-PUBLISH_PATH = os.path.join(PLUGINS_DIR, "publish")
-LOAD_PATH = os.path.join(PLUGINS_DIR, "load")
-CREATE_PATH = os.path.join(PLUGINS_DIR, "create")
-
-
-def application_launch():
- pass
+from openpype.host import HostBase
+from openpype.hosts.webpublisher import WEBPUBLISHER_ROOT_DIR
-
-def install():
- print("Installing Pype config...")
-
- pyblish.register_plugin_path(PUBLISH_PATH)
- avalon.register_plugin_path(avalon.Loader, LOAD_PATH)
- avalon.register_plugin_path(avalon.Creator, CREATE_PATH)
- log.info(PUBLISH_PATH)
-
- io.install()
- avalon.on("application.launched", application_launch)
+log = logging.getLogger("openpype.hosts.webpublisher")
-def uninstall():
- pyblish.deregister_plugin_path(PUBLISH_PATH)
- avalon.deregister_plugin_path(avalon.Loader, LOAD_PATH)
- avalon.deregister_plugin_path(avalon.Creator, CREATE_PATH)
+class WebpublisherHost(HostBase):
+ name = "webpublisher"
+ def install(self):
+ print("Installing Pype config...")
+ pyblish.api.register_host(self.name)
-# to have required methods for interface
-def ls():
- pass
+ publish_plugin_dir = os.path.join(
+ WEBPUBLISHER_ROOT_DIR, "plugins", "publish"
+ )
+ pyblish.api.register_plugin_path(publish_plugin_dir)
+ self.log.info(publish_plugin_dir)
diff --git a/openpype/hosts/webpublisher/lib.py b/openpype/hosts/webpublisher/lib.py
new file mode 100644
index 00000000000..4bc3f1db80d
--- /dev/null
+++ b/openpype/hosts/webpublisher/lib.py
@@ -0,0 +1,269 @@
+import os
+from datetime import datetime
+import collections
+import json
+
+from bson.objectid import ObjectId
+
+import pyblish.util
+import pyblish.api
+
+from openpype.client.mongo import OpenPypeMongoConnection
+from openpype.settings import get_project_settings
+from openpype.lib import Logger
+from openpype.lib.profiles_filtering import filter_profiles
+from openpype.pipeline.publish.lib import find_close_plugin
+
+ERROR_STATUS = "error"
+IN_PROGRESS_STATUS = "in_progress"
+REPROCESS_STATUS = "reprocess"
+SENT_REPROCESSING_STATUS = "sent_for_reprocessing"
+FINISHED_REPROCESS_STATUS = "republishing_finished"
+FINISHED_OK_STATUS = "finished_ok"
+
+log = Logger.get_logger(__name__)
+
+
+def parse_json(path):
+ """Parses json file at 'path' location
+
+ Returns:
+ (dict) or None if unparsable
+ Raises:
+ AsssertionError if 'path' doesn't exist
+ """
+ path = path.strip('\"')
+ assert os.path.isfile(path), (
+ "Path to json file doesn't exist. \"{}\"".format(path)
+ )
+ data = None
+ with open(path, "r") as json_file:
+ try:
+ data = json.load(json_file)
+ except Exception as exc:
+ log.error(
+ "Error loading json: {} - Exception: {}".format(path, exc)
+ )
+ return data
+
+
+def get_batch_asset_task_info(ctx):
+ """Parses context data from webpublisher's batch metadata
+
+ Returns:
+ (tuple): asset, task_name (Optional), task_type
+ """
+ task_type = "default_task_type"
+ task_name = None
+ asset = None
+
+ if ctx["type"] == "task":
+ items = ctx["path"].split('/')
+ asset = items[-2]
+ task_name = ctx["name"]
+ task_type = ctx["attributes"]["type"]
+ else:
+ asset = ctx["name"]
+
+ return asset, task_name, task_type
+
+
+def get_webpublish_conn():
+ """Get connection to OP 'webpublishes' collection."""
+ mongo_client = OpenPypeMongoConnection.get_mongo_client()
+ database_name = os.environ["OPENPYPE_DATABASE_NAME"]
+ return mongo_client[database_name]["webpublishes"]
+
+
+def start_webpublish_log(dbcon, batch_id, user):
+ """Start new log record for 'batch_id'
+
+ Args:
+ dbcon (OpenPypeMongoConnection)
+ batch_id (str)
+ user (str)
+ Returns
+ (ObjectId) from DB
+ """
+ return dbcon.insert_one({
+ "batch_id": batch_id,
+ "start_date": datetime.now(),
+ "user": user,
+ "status": IN_PROGRESS_STATUS,
+ "progress": 0 # integer 0-100, percentage
+ }).inserted_id
+
+
+def publish_and_log(dbcon, _id, log, close_plugin_name=None, batch_id=None):
+ """Loops through all plugins, logs ok and fails into OP DB.
+
+ Args:
+ dbcon (OpenPypeMongoConnection)
+ _id (str) - id of current job in DB
+ log (openpype.lib.Logger)
+ batch_id (str) - id sent from frontend
+ close_plugin_name (str): name of plugin with responsibility to
+ close host app
+ """
+ # Error exit as soon as any error occurs.
+ error_format = "Failed {plugin.__name__}: {error} -- {error.traceback}\n"
+ error_format += "-" * 80 + "\n"
+
+ close_plugin = find_close_plugin(close_plugin_name, log)
+
+ if isinstance(_id, str):
+ _id = ObjectId(_id)
+
+ log_lines = []
+ processed = 0
+ log_every = 5
+ for result in pyblish.util.publish_iter():
+ for record in result["records"]:
+ log_lines.append("{}: {}".format(
+ result["plugin"].label, record.msg))
+ processed += 1
+
+ if result["error"]:
+ log.error(error_format.format(**result))
+ log_lines = [error_format.format(**result)] + log_lines
+ dbcon.update_one(
+ {"_id": _id},
+ {"$set":
+ {
+ "finish_date": datetime.now(),
+ "status": ERROR_STATUS,
+ "log": os.linesep.join(log_lines)
+
+ }}
+ )
+ if close_plugin: # close host app explicitly after error
+ context = pyblish.api.Context()
+ close_plugin().process(context)
+ return
+ elif processed % log_every == 0:
+ # pyblish returns progress in 0.0 - 2.0
+ progress = min(round(result["progress"] / 2 * 100), 99)
+ dbcon.update_one(
+ {"_id": _id},
+ {"$set":
+ {
+ "progress": progress,
+ "log": os.linesep.join(log_lines)
+ }}
+ )
+
+ # final update
+ if batch_id:
+ dbcon.update_many(
+ {"batch_id": batch_id, "status": SENT_REPROCESSING_STATUS},
+ {
+ "$set":
+ {
+ "finish_date": datetime.now(),
+ "status": FINISHED_REPROCESS_STATUS,
+ }
+ }
+ )
+
+ dbcon.update_one(
+ {"_id": _id},
+ {
+ "$set":
+ {
+ "finish_date": datetime.now(),
+ "status": FINISHED_OK_STATUS,
+ "progress": 100,
+ "log": os.linesep.join(log_lines)
+ }
+ }
+ )
+
+
+def fail_batch(_id, dbcon, msg):
+ """Set current batch as failed as there is some problem.
+
+ Raises:
+ ValueError
+ """
+ dbcon.update_one(
+ {"_id": _id},
+ {"$set":
+ {
+ "finish_date": datetime.now(),
+ "status": ERROR_STATUS,
+ "log": msg
+
+ }}
+ )
+ raise ValueError(msg)
+
+
+def find_variant_key(application_manager, host):
+ """Searches for latest installed variant for 'host'
+
+ Args:
+ application_manager (ApplicationManager)
+ host (str)
+ Returns
+ (string) (optional)
+ Raises:
+ (ValueError) if no variant found
+ """
+ app_group = application_manager.app_groups.get(host)
+ if not app_group or not app_group.enabled:
+ raise ValueError("No application {} configured".format(host))
+
+ found_variant_key = None
+ # finds most up-to-date variant if any installed
+ sorted_variants = collections.OrderedDict(
+ sorted(app_group.variants.items()))
+ for variant_key, variant in sorted_variants.items():
+ for executable in variant.executables:
+ if executable.exists():
+ found_variant_key = variant_key
+
+ if not found_variant_key:
+ raise ValueError("No executable for {} found".format(host))
+
+ return found_variant_key
+
+
+def get_task_data(batch_dir):
+ """Return parsed data from first task manifest.json
+
+ Used for `remotepublishfromapp` command where batch contains only
+ single task with publishable workfile.
+
+ Returns:
+ (dict)
+ Throws:
+ (ValueError) if batch or task manifest not found or broken
+ """
+ batch_data = parse_json(os.path.join(batch_dir, "manifest.json"))
+ if not batch_data:
+ raise ValueError(
+ "Cannot parse batch meta in {} folder".format(batch_dir))
+ task_dir_name = batch_data["tasks"][0]
+ task_data = parse_json(os.path.join(batch_dir, task_dir_name,
+ "manifest.json"))
+ if not task_data:
+ raise ValueError(
+ "Cannot parse batch meta in {} folder".format(task_data))
+
+ return task_data
+
+
+def get_timeout(project_name, host_name, task_type):
+ """Returns timeout(seconds) from Setting profile."""
+ filter_data = {
+ "task_types": task_type,
+ "hosts": host_name
+ }
+ timeout_profiles = (get_project_settings(project_name)["webpublisher"]
+ ["timeout_profiles"])
+ matching_item = filter_profiles(timeout_profiles, filter_data)
+ timeout = 3600
+ if matching_item:
+ timeout = matching_item["timeout"]
+
+ return timeout
diff --git a/openpype/hosts/webpublisher/plugins/publish/collect_batch_data.py b/openpype/hosts/webpublisher/plugins/publish/collect_batch_data.py
index a710fcb3e82..eb2737b2763 100644
--- a/openpype/hosts/webpublisher/plugins/publish/collect_batch_data.py
+++ b/openpype/hosts/webpublisher/plugins/publish/collect_batch_data.py
@@ -1,18 +1,25 @@
-"""Loads batch context from json and continues in publish process.
+"""Parses batch context from json and continues in publish process.
Provides:
context -> Loaded batch file.
+ - asset
+ - task (task name)
+ - taskType
+ - project_name
+ - variant
"""
import os
import pyblish.api
-from avalon import io
-from openpype.lib.plugin_tools import (
+
+from openpype.pipeline import legacy_io
+from openpype_modules.webpublisher.lib import (
parse_json,
- get_batch_asset_task_info
+ get_batch_asset_task_info,
+ get_webpublish_conn,
+ IN_PROGRESS_STATUS
)
-from openpype.lib.remote_publish import get_webpublish_conn
class CollectBatchData(pyblish.api.ContextPlugin):
@@ -24,7 +31,7 @@ class CollectBatchData(pyblish.api.ContextPlugin):
# must be really early, context values are only in json file
order = pyblish.api.CollectorOrder - 0.495
label = "Collect batch data"
- host = ["webpublisher"]
+ hosts = ["webpublisher"]
def process(self, context):
batch_dir = os.environ.get("OPENPYPE_PUBLISH_DATA")
@@ -52,13 +59,15 @@ def process(self, context):
)
os.environ["AVALON_ASSET"] = asset_name
- io.Session["AVALON_ASSET"] = asset_name
+ legacy_io.Session["AVALON_ASSET"] = asset_name
os.environ["AVALON_TASK"] = task_name
- io.Session["AVALON_TASK"] = task_name
+ legacy_io.Session["AVALON_TASK"] = task_name
context.data["asset"] = asset_name
context.data["task"] = task_name
context.data["taskType"] = task_type
+ context.data["project_name"] = project_name
+ context.data["variant"] = batch_data["variant"]
self._set_ctx_path(batch_data)
@@ -74,7 +83,7 @@ def _set_ctx_path(self, batch_data):
dbcon.update_one(
{
"batch_id": batch_id,
- "status": "in_progress"
+ "status": IN_PROGRESS_STATUS
},
{
"$set": {
diff --git a/openpype/hosts/webpublisher/plugins/publish/collect_published_files.py b/openpype/hosts/webpublisher/plugins/publish/collect_published_files.py
index d2754b3df38..278a102f9d0 100644
--- a/openpype/hosts/webpublisher/plugins/publish/collect_published_files.py
+++ b/openpype/hosts/webpublisher/plugins/publish/collect_published_files.py
@@ -10,10 +10,21 @@
import os
import clique
import tempfile
-from avalon import io
+import math
+
import pyblish.api
-from openpype.lib import prepare_template_data
-from openpype.lib.plugin_tools import parse_json
+
+from openpype.client import (
+ get_asset_by_name,
+ get_last_version_by_subset_name
+)
+from openpype.lib import (
+ prepare_template_data,
+ get_ffprobe_streams,
+ convert_ffprobe_fps_value,
+)
+from openpype.pipeline.create import get_subset_name
+from openpype_modules.webpublisher.lib import parse_json
class CollectPublishedFiles(pyblish.api.ContextPlugin):
@@ -21,15 +32,20 @@ class CollectPublishedFiles(pyblish.api.ContextPlugin):
This collector will try to find json files in provided
`OPENPYPE_PUBLISH_DATA`. Those files _MUST_ share same context.
+ This covers 'basic' webpublishes, eg artists uses Standalone Publisher to
+ publish rendered frames or assets.
+
+ This is not applicable for 'studio' processing where host application is
+ called to process uploaded workfile and render frames itself.
"""
# must be really early, context values are only in json file
order = pyblish.api.CollectorOrder - 0.490
label = "Collect rendered frames"
- host = ["webpublisher"]
+ hosts = ["webpublisher"]
targets = ["filespublish"]
# from Settings
- task_type_to_family = {}
+ task_type_to_family = []
def process(self, context):
batch_dir = context.data["batchDir"]
@@ -41,9 +57,13 @@ def process(self, context):
self.log.info("task_sub:: {}".format(task_subfolders))
+ project_name = context.data["project_name"]
asset_name = context.data["asset"]
+ asset_doc = get_asset_by_name(project_name, asset_name)
task_name = context.data["task"]
task_type = context.data["taskType"]
+ project_name = context.data["project_name"]
+ variant = context.data["variant"]
for task_dir in task_subfolders:
task_data = parse_json(os.path.join(task_dir,
"manifest.json"))
@@ -52,20 +72,28 @@ def process(self, context):
is_sequence = len(task_data["files"]) > 1
_, extension = os.path.splitext(task_data["files"][0])
- family, families, subset_template, tags = self._get_family(
+ family, families, tags = self._get_family(
self.task_type_to_family,
task_type,
is_sequence,
extension.replace(".", ''))
- subset = self._get_subset_name(
- family, subset_template, task_name, task_data["variant"]
+ subset_name = get_subset_name(
+ family,
+ variant,
+ task_name,
+ asset_doc,
+ project_name=project_name,
+ host_name="webpublisher",
+ project_settings=context.data["project_settings"]
+ )
+ version = self._get_next_version(
+ project_name, asset_doc, subset_name
)
- version = self._get_last_version(asset_name, subset) + 1
- instance = context.create_instance(subset)
+ instance = context.create_instance(subset_name)
instance.data["asset"] = asset_name
- instance.data["subset"] = subset
+ instance.data["subset"] = subset_name
instance.data["family"] = family
instance.data["families"] = families
instance.data["version"] = version
@@ -84,11 +112,30 @@ def process(self, context):
instance.data["frameEnd"] = \
instance.data["representations"][0]["frameEnd"]
else:
- instance.data["frameStart"] = 0
- instance.data["frameEnd"] = 1
+ frame_start = asset_doc["data"]["frameStart"]
+ instance.data["frameStart"] = frame_start
+ instance.data["frameEnd"] = asset_doc["data"]["frameEnd"]
instance.data["representations"] = self._get_single_repre(
task_dir, task_data["files"], tags
)
+ if family != 'workfile':
+ file_url = os.path.join(task_dir, task_data["files"][0])
+ try:
+ no_of_frames = self._get_number_of_frames(file_url)
+ if no_of_frames:
+ frame_end = int(frame_start) + \
+ math.ceil(no_of_frames)
+ frame_end = math.ceil(frame_end) - 1
+ instance.data["frameEnd"] = frame_end
+ self.log.debug("frameEnd:: {}".format(
+ instance.data["frameEnd"]))
+ except Exception:
+ self.log.warning("Unable to count frames "
+ "duration {}".format(no_of_frames))
+
+ # raise ValueError("STOP")
+ instance.data["handleStart"] = asset_doc["data"]["handleStart"]
+ instance.data["handleEnd"] = asset_doc["data"]["handleEnd"]
self.log.info("instance.data:: {}".format(instance.data))
@@ -114,7 +161,7 @@ def _get_single_repre(self, task_dir, files, tags):
return [repre_data]
def _process_sequence(self, files, task_dir, tags):
- """Prepare reprentations for sequence of files."""
+ """Prepare representation for sequence of files."""
collections, remainder = clique.assemble(files)
assert len(collections) == 1, \
"Too many collections in {}".format(files)
@@ -144,7 +191,7 @@ def _get_family(self, settings, task_type, is_sequence, extension):
extension (str): without '.'
Returns:
- (family, [families], subset_template_name, tags) tuple
+ (family, [families], tags) tuple
AssertionError if not matching family found
"""
task_type = task_type.lower()
@@ -155,12 +202,21 @@ def _get_family(self, settings, task_type, is_sequence, extension):
assert task_obj, "No family configuration for '{}'".format(task_type)
found_family = None
- for family, content in task_obj.items():
- if is_sequence != content["is_sequence"]:
+ families_config = []
+ # backward compatibility, should be removed pretty soon
+ if isinstance(task_obj, dict):
+ for family, config in task_obj:
+ config["result_family"] = family
+ families_config.append(config)
+ else:
+ families_config = task_obj
+
+ for config in families_config:
+ if is_sequence != config["is_sequence"]:
continue
- if extension in content["extensions"] or \
- '' in content["extensions"]: # all extensions setting
- found_family = family
+ if (extension in config["extensions"] or
+ '' in config["extensions"]): # all extensions setting
+ found_family = config["result_family"]
break
msg = "No family found for combination of " +\
@@ -168,57 +224,60 @@ def _get_family(self, settings, task_type, is_sequence, extension):
task_type, is_sequence, extension)
assert found_family, msg
- return found_family, \
- content["families"], \
- content["subset_template_name"], \
- content["tags"]
-
- def _get_last_version(self, asset_name, subset_name):
- """Returns version number or 0 for 'asset' and 'subset'"""
- query = [
- {
- "$match": {"type": "asset", "name": asset_name}
- },
- {
- "$lookup":
- {
- "from": os.environ["AVALON_PROJECT"],
- "localField": "_id",
- "foreignField": "parent",
- "as": "subsets"
- }
- },
- {
- "$unwind": "$subsets"
- },
- {
- "$match": {"subsets.type": "subset",
- "subsets.name": subset_name}},
- {
- "$lookup":
- {
- "from": os.environ["AVALON_PROJECT"],
- "localField": "subsets._id",
- "foreignField": "parent",
- "as": "versions"
- }
- },
- {
- "$unwind": "$versions"
- },
- {
- "$group": {
- "_id": {
- "asset_name": "$name",
- "subset_name": "$subsets.name"
- },
- 'version': {'$max': "$versions.name"}
- }
- }
- ]
- version = list(io.aggregate(query))
-
- if version:
- return version[0].get("version") or 0
- else:
- return 0
+ return (found_family,
+ config["families"],
+ config["tags"])
+
+ def _get_next_version(self, project_name, asset_doc, subset_name):
+ """Returns version number or 1 for 'asset' and 'subset'"""
+
+ version_doc = get_last_version_by_subset_name(
+ project_name,
+ subset_name,
+ asset_doc["_id"],
+ fields=["name"]
+ )
+ version = 1
+ if version_doc:
+ version += int(version_doc["name"])
+ return version
+
+ def _get_number_of_frames(self, file_url):
+ """Return duration in frames"""
+ try:
+ streams = get_ffprobe_streams(file_url, self.log)
+ except Exception as exc:
+ raise AssertionError((
+ "FFprobe couldn't read information about input file: \"{}\"."
+ " Error message: {}"
+ ).format(file_url, str(exc)))
+
+ first_video_stream = None
+ for stream in streams:
+ if "width" in stream and "height" in stream:
+ first_video_stream = stream
+ break
+
+ if first_video_stream:
+ nb_frames = stream.get("nb_frames")
+ if nb_frames:
+ try:
+ return int(nb_frames)
+ except ValueError:
+ self.log.warning(
+ "nb_frames {} not convertible".format(nb_frames))
+
+ duration = stream.get("duration")
+ frame_rate = convert_ffprobe_fps_value(
+ stream.get("r_frame_rate", '0/0')
+ )
+ self.log.debug("duration:: {} frame_rate:: {}".format(
+ duration, frame_rate))
+ try:
+ return float(duration) * float(frame_rate)
+ except ValueError:
+ self.log.warning(
+ "{} or {} cannot be converted".format(duration,
+ frame_rate))
+
+ self.log.warning("Cannot get number of frames")
diff --git a/openpype/hosts/webpublisher/plugins/publish/collect_tvpaint_instances.py b/openpype/hosts/webpublisher/plugins/publish/collect_tvpaint_instances.py
index 976a14e808e..948e86c23ee 100644
--- a/openpype/hosts/webpublisher/plugins/publish/collect_tvpaint_instances.py
+++ b/openpype/hosts/webpublisher/plugins/publish/collect_tvpaint_instances.py
@@ -10,7 +10,7 @@
import copy
import pyblish.api
-from openpype.lib import get_subset_name_with_asset_doc
+from openpype.pipeline.create import get_subset_name
class CollectTVPaintInstances(pyblish.api.ContextPlugin):
@@ -28,7 +28,7 @@ class CollectTVPaintInstances(pyblish.api.ContextPlugin):
render_layer_pass_name = "beauty"
# Set by settings
- # Regex must constain 'layer' and 'variant' groups which are extracted from
+ # Regex must contain 'layer' and 'variant' groups which are extracted from
# name when instances are created
layer_name_regex = r"(?PL[0-9]{3}_\w+)_(?P.+)"
@@ -47,13 +47,14 @@ def process(self, context):
new_instances = []
# Workfile instance
- workfile_subset_name = get_subset_name_with_asset_doc(
+ workfile_subset_name = get_subset_name(
self.workfile_family,
self.workfile_variant,
task_name,
asset_doc,
project_name,
- host_name
+ host_name,
+ project_settings=context.data["project_settings"]
)
workfile_instance = self._create_workfile_instance(
context, workfile_subset_name
@@ -61,13 +62,14 @@ def process(self, context):
new_instances.append(workfile_instance)
# Review instance
- review_subset_name = get_subset_name_with_asset_doc(
+ review_subset_name = get_subset_name(
self.review_family,
self.review_variant,
task_name,
asset_doc,
project_name,
- host_name
+ host_name,
+ project_settings=context.data["project_settings"]
)
review_instance = self._create_review_instance(
context, review_subset_name
@@ -114,14 +116,15 @@ def process(self, context):
"family": "render"
}
- subset_name = get_subset_name_with_asset_doc(
+ subset_name = get_subset_name(
self.render_pass_family,
render_pass,
task_name,
asset_doc,
project_name,
host_name,
- dynamic_data=dynamic_data
+ dynamic_data=dynamic_data,
+ project_settings=context.data["project_settings"]
)
instance = self._create_render_pass_instance(
@@ -137,14 +140,15 @@ def process(self, context):
# Override family for subset name
"family": "render"
}
- subset_name = get_subset_name_with_asset_doc(
+ subset_name = get_subset_name(
self.render_layer_family,
variant,
task_name,
asset_doc,
project_name,
host_name,
- dynamic_data=dynamic_data
+ dynamic_data=dynamic_data,
+ project_settings=context.data["project_settings"]
)
instance = self._create_render_layer_instance(
context, layers, subset_name
diff --git a/openpype/hosts/webpublisher/plugins/publish/collect_tvpaint_workfile_data.py b/openpype/hosts/webpublisher/plugins/publish/collect_tvpaint_workfile_data.py
index f0f29260a2e..b5f8ed9c8ff 100644
--- a/openpype/hosts/webpublisher/plugins/publish/collect_tvpaint_workfile_data.py
+++ b/openpype/hosts/webpublisher/plugins/publish/collect_tvpaint_workfile_data.py
@@ -16,11 +16,11 @@
import json
import shutil
import pyblish.api
-from openpype.lib.plugin_tools import parse_json
from openpype.hosts.tvpaint.worker import (
SenderTVPaintCommands,
CollectSceneData
)
+from openpype_modules.webpublisher.lib import parse_json
class CollectTVPaintWorkfileData(pyblish.api.ContextPlugin):
diff --git a/openpype/hosts/webpublisher/plugins/publish/extract_thumbnail.py b/openpype/hosts/webpublisher/plugins/publish/extract_thumbnail.py
new file mode 100644
index 00000000000..a56521891bf
--- /dev/null
+++ b/openpype/hosts/webpublisher/plugins/publish/extract_thumbnail.py
@@ -0,0 +1,137 @@
+import os
+import shutil
+
+import pyblish.api
+from openpype.lib import (
+ get_ffmpeg_tool_path,
+
+ run_subprocess,
+
+ get_transcode_temp_directory,
+ convert_input_paths_for_ffmpeg,
+ should_convert_for_ffmpeg
+)
+
+
+class ExtractThumbnail(pyblish.api.InstancePlugin):
+ """Create jpg thumbnail from input using ffmpeg."""
+
+ label = "Extract Thumbnail"
+ order = pyblish.api.ExtractorOrder
+ families = [
+ "render",
+ "image"
+ ]
+ hosts = ["webpublisher"]
+ targets = ["filespublish"]
+
+ def process(self, instance):
+ self.log.info("subset {}".format(instance.data['subset']))
+
+ filtered_repres = self._get_filtered_repres(instance)
+ for repre in filtered_repres:
+ repre_files = repre["files"]
+ if not isinstance(repre_files, (list, tuple)):
+ input_file = repre_files
+ else:
+ file_index = int(float(len(repre_files)) * 0.5)
+ input_file = repre_files[file_index]
+
+ stagingdir = os.path.normpath(repre["stagingDir"])
+
+ full_input_path = os.path.join(stagingdir, input_file)
+ self.log.info("Input filepath: {}".format(full_input_path))
+
+ do_convert = should_convert_for_ffmpeg(full_input_path)
+ # If result is None the requirement of conversion can't be
+ # determined
+ if do_convert is None:
+ self.log.info((
+ "Can't determine if representation requires conversion."
+ " Skipped."
+ ))
+ continue
+
+ # Do conversion if needed
+ # - change staging dir of source representation
+ # - must be set back after output definitions processing
+ convert_dir = None
+ if do_convert:
+ convert_dir = get_transcode_temp_directory()
+ filename = os.path.basename(full_input_path)
+ convert_input_paths_for_ffmpeg(
+ [full_input_path],
+ convert_dir,
+ self.log
+ )
+ full_input_path = os.path.join(convert_dir, filename)
+
+ filename = os.path.splitext(input_file)[0]
+ while filename.endswith("."):
+ filename = filename[:-1]
+ thumbnail_filename = filename + "_thumbnail.jpg"
+ full_output_path = os.path.join(stagingdir, thumbnail_filename)
+
+ self.log.info("output {}".format(full_output_path))
+
+ ffmpeg_args = [
+ get_ffmpeg_tool_path("ffmpeg"),
+ "-y",
+ "-i", full_input_path,
+ "-vframes", "1",
+ full_output_path
+ ]
+
+ # run subprocess
+ self.log.debug("{}".format(" ".join(ffmpeg_args)))
+ try: # temporary until oiiotool is supported cross platform
+ run_subprocess(
+ ffmpeg_args, logger=self.log
+ )
+ except RuntimeError as exp:
+ if "Compression" in str(exp):
+ self.log.debug(
+ "Unsupported compression on input files. Skipping!!!"
+ )
+ return
+ self.log.warning("Conversion crashed", exc_info=True)
+ raise
+
+ new_repre = {
+ "name": "thumbnail",
+ "ext": "jpg",
+ "files": thumbnail_filename,
+ "stagingDir": stagingdir,
+ "thumbnail": True,
+ "tags": ["thumbnail"]
+ }
+
+ # adding representation
+ self.log.debug("Adding: {}".format(new_repre))
+ instance.data["representations"].append(new_repre)
+
+ # Cleanup temp folder
+ if convert_dir is not None and os.path.exists(convert_dir):
+ shutil.rmtree(convert_dir)
+
+ def _get_filtered_repres(self, instance):
+ filtered_repres = []
+ repres = instance.data.get("representations") or []
+ for repre in repres:
+ self.log.debug(repre)
+ tags = repre.get("tags") or []
+ # Skip instance if already has thumbnail representation
+ if "thumbnail" in tags:
+ return []
+
+ if "review" not in tags:
+ continue
+
+ if not repre.get("files"):
+ self.log.info((
+ "Representation \"{}\" don't have files. Skipping"
+ ).format(repre["name"]))
+ continue
+
+ filtered_repres.append(repre)
+ return filtered_repres
diff --git a/openpype/hosts/webpublisher/plugins/publish/extract_tvpaint_workfile.py b/openpype/hosts/webpublisher/plugins/publish/extract_tvpaint_workfile.py
index 85c8526c83c..2142d740a58 100644
--- a/openpype/hosts/webpublisher/plugins/publish/extract_tvpaint_workfile.py
+++ b/openpype/hosts/webpublisher/plugins/publish/extract_tvpaint_workfile.py
@@ -286,7 +286,7 @@ def _post_render_processing(self, post_render_data, mark_in, mark_out):
if single_file:
repre_files = repre_files[0]
- # Extension is harcoded
+ # Extension is hardcoded
# - changing extension would require change code
new_repre = {
"name": "png",
@@ -407,7 +407,7 @@ def add_render_command(
mark_out (int): On which frame index export will end.
layers (list): List of layers to be exported.
- Retruns:
+ Returns:
tuple: With 2 items first is list of filenames second is path to
thumbnail.
"""
diff --git a/openpype/hosts/webpublisher/publish_functions.py b/openpype/hosts/webpublisher/publish_functions.py
new file mode 100644
index 00000000000..83f53ced68b
--- /dev/null
+++ b/openpype/hosts/webpublisher/publish_functions.py
@@ -0,0 +1,205 @@
+import os
+import time
+import pyblish.api
+import pyblish.util
+
+from openpype.lib import Logger
+from openpype.lib.applications import (
+ ApplicationManager,
+ get_app_environments_for_context,
+)
+from openpype.pipeline import install_host
+from openpype.hosts.webpublisher.api import WebpublisherHost
+
+from .lib import (
+ get_batch_asset_task_info,
+ get_webpublish_conn,
+ start_webpublish_log,
+ publish_and_log,
+ fail_batch,
+ find_variant_key,
+ get_task_data,
+ get_timeout,
+ IN_PROGRESS_STATUS
+)
+
+
+def cli_publish(project_name, batch_path, user_email, targets):
+ """Start headless publishing.
+
+ Used to publish rendered assets, workfiles etc via Webpublisher.
+ Eventually should be yanked out to Webpublisher cli.
+
+ Publish use json from passed paths argument.
+
+ Args:
+ project_name (str): project to publish (only single context is
+ expected per call of remotepublish
+ batch_path (str): Path batch folder. Contains subfolders with
+ resources (workfile, another subfolder 'renders' etc.)
+ user_email (string): email address for webpublisher - used to
+ find Ftrack user with same email
+ targets (list): Pyblish targets
+ (to choose validator for example)
+
+ Raises:
+ RuntimeError: When there is no path to process.
+ """
+
+ if not batch_path:
+ raise RuntimeError("No publish paths specified")
+
+ log = Logger.get_logger("remotepublish")
+ log.info("remotepublish command")
+
+ # Register target and host
+ webpublisher_host = WebpublisherHost()
+
+ os.environ["OPENPYPE_PUBLISH_DATA"] = batch_path
+ os.environ["AVALON_PROJECT"] = project_name
+ os.environ["AVALON_APP"] = webpublisher_host.name
+ os.environ["USER_EMAIL"] = user_email
+ os.environ["HEADLESS_PUBLISH"] = 'true' # to use in app lib
+
+ if targets:
+ if isinstance(targets, str):
+ targets = [targets]
+ for target in targets:
+ pyblish.api.register_target(target)
+
+ install_host(webpublisher_host)
+
+ log.info("Running publish ...")
+
+ _, batch_id = os.path.split(batch_path)
+ dbcon = get_webpublish_conn()
+ _id = start_webpublish_log(dbcon, batch_id, user_email)
+
+ task_data = get_task_data(batch_path)
+ if not task_data["context"]:
+ msg = "Batch manifest must contain context data"
+ msg += "Create new batch and set context properly."
+ fail_batch(_id, dbcon, msg)
+
+ publish_and_log(dbcon, _id, log, batch_id=batch_id)
+
+ log.info("Publish finished.")
+
+
+def cli_publish_from_app(
+ project_name, batch_path, host_name, user_email, targets
+):
+ """Opens installed variant of 'host' and run remote publish there.
+
+ Eventually should be yanked out to Webpublisher cli.
+
+ Currently implemented and tested for Photoshop where customer
+ wants to process uploaded .psd file and publish collected layers
+ from there. Triggered by Webpublisher.
+
+ Checks if no other batches are running (status =='in_progress). If
+ so, it sleeps for SLEEP (this is separate process),
+ waits for WAIT_FOR seconds altogether.
+
+ Requires installed host application on the machine.
+
+ Runs publish process as user would, in automatic fashion.
+
+ Args:
+ project_name (str): project to publish (only single context is
+ expected per call of remotepublish
+ batch_path (str): Path batch folder. Contains subfolders with
+ resources (workfile, another subfolder 'renders' etc.)
+ host_name (str): 'photoshop'
+ user_email (string): email address for webpublisher - used to
+ find Ftrack user with same email
+ targets (list): Pyblish targets
+ (to choose validator for example)
+ """
+
+ log = Logger.get_logger("RemotePublishFromApp")
+
+ log.info("remotepublishphotoshop command")
+
+ task_data = get_task_data(batch_path)
+
+ workfile_path = os.path.join(batch_path,
+ task_data["task"],
+ task_data["files"][0])
+
+ print("workfile_path {}".format(workfile_path))
+
+ batch_id = task_data["batch"]
+ dbcon = get_webpublish_conn()
+ # safer to start logging here, launch might be broken altogether
+ _id = start_webpublish_log(dbcon, batch_id, user_email)
+
+ batches_in_progress = list(dbcon.find({"status": IN_PROGRESS_STATUS}))
+ if len(batches_in_progress) > 1:
+ running_batches = [str(batch["_id"])
+ for batch in batches_in_progress
+ if batch["_id"] != _id]
+ msg = "There are still running batches {}\n". \
+ format("\n".join(running_batches))
+ msg += "Ask admin to check them and reprocess current batch"
+ fail_batch(_id, dbcon, msg)
+
+ if not task_data["context"]:
+ msg = "Batch manifest must contain context data"
+ msg += "Create new batch and set context properly."
+ fail_batch(_id, dbcon, msg)
+
+ asset_name, task_name, task_type = get_batch_asset_task_info(
+ task_data["context"])
+
+ application_manager = ApplicationManager()
+ found_variant_key = find_variant_key(application_manager, host_name)
+ app_name = "{}/{}".format(host_name, found_variant_key)
+
+ # must have for proper launch of app
+ env = get_app_environments_for_context(
+ project_name,
+ asset_name,
+ task_name,
+ app_name
+ )
+ print("env:: {}".format(env))
+ os.environ.update(env)
+
+ os.environ["OPENPYPE_PUBLISH_DATA"] = batch_path
+ # must pass identifier to update log lines for a batch
+ os.environ["BATCH_LOG_ID"] = str(_id)
+ os.environ["HEADLESS_PUBLISH"] = 'true' # to use in app lib
+ os.environ["USER_EMAIL"] = user_email
+
+ pyblish.api.register_host(host_name)
+ if targets:
+ if isinstance(targets, str):
+ targets = [targets]
+ current_targets = os.environ.get("PYBLISH_TARGETS", "").split(
+ os.pathsep)
+ for target in targets:
+ current_targets.append(target)
+
+ os.environ["PYBLISH_TARGETS"] = os.pathsep.join(
+ set(current_targets))
+
+ data = {
+ "last_workfile_path": workfile_path,
+ "start_last_workfile": True,
+ "project_name": project_name,
+ "asset_name": asset_name,
+ "task_name": task_name
+ }
+
+ launched_app = application_manager.launch(app_name, **data)
+
+ timeout = get_timeout(project_name, host_name, task_type)
+
+ time_start = time.time()
+ while launched_app.poll() is None:
+ time.sleep(0.5)
+ if time.time() - time_start > timeout:
+ launched_app.terminate()
+ msg = "Timeout reached"
+ fail_batch(_id, dbcon, msg)
diff --git a/openpype/hosts/webpublisher/webserver_service/__init__.py b/openpype/hosts/webpublisher/webserver_service/__init__.py
new file mode 100644
index 00000000000..73111d286e2
--- /dev/null
+++ b/openpype/hosts/webpublisher/webserver_service/__init__.py
@@ -0,0 +1,6 @@
+from .webserver import run_webserver
+
+
+__all__ = (
+ "run_webserver",
+)
diff --git a/openpype/hosts/webpublisher/webserver_service/webpublish_routes.py b/openpype/hosts/webpublisher/webserver_service/webpublish_routes.py
index 30399a6ba76..4039d2c8ecd 100644
--- a/openpype/hosts/webpublisher/webserver_service/webpublish_routes.py
+++ b/openpype/hosts/webpublisher/webserver_service/webpublish_routes.py
@@ -2,38 +2,46 @@
import os
import json
import datetime
-from bson.objectid import ObjectId
import collections
-from aiohttp.web_response import Response
import subprocess
+from bson.objectid import ObjectId
+from aiohttp.web_response import Response
-from avalon.api import AvalonMongoDB
-
-from openpype.lib import OpenPypeMongoConnection
-from openpype_modules.avalon_apps.rest_api import _RestApiEndpoint
-from openpype.lib.remote_publish import get_task_data
+from openpype.client import (
+ get_projects,
+ get_assets,
+)
+from openpype.lib import Logger
from openpype.settings import get_project_settings
+from openpype_modules.webserver.base_routes import RestApiEndpoint
+from openpype_modules.webpublisher import WebpublisherAddon
+from openpype_modules.webpublisher.lib import (
+ get_webpublish_conn,
+ get_task_data,
+ ERROR_STATUS,
+ REPROCESS_STATUS
+)
-from openpype.lib import PypeLogger
+log = Logger.get_logger("WebpublishRoutes")
-log = PypeLogger.get_logger("WebServer")
+class ResourceRestApiEndpoint(RestApiEndpoint):
+ def __init__(self, resource):
+ self.resource = resource
+ super(ResourceRestApiEndpoint, self).__init__()
-class RestApiResource:
- """Resource carrying needed info and Avalon DB connection for publish."""
- def __init__(self, server_manager, executable, upload_dir,
- studio_task_queue=None):
- self.server_manager = server_manager
- self.upload_dir = upload_dir
- self.executable = executable
- if studio_task_queue is None:
- studio_task_queue = collections.deque().dequeu
- self.studio_task_queue = studio_task_queue
+class WebpublishApiEndpoint(ResourceRestApiEndpoint):
+ @property
+ def dbcon(self):
+ return self.resource.dbcon
+
- self.dbcon = AvalonMongoDB()
- self.dbcon.install()
+class JsonApiResource:
+ """Resource for json manipulation.
+ All resources handling sending output to REST should inherit from
+ """
@staticmethod
def json_dump_handler(value):
if isinstance(value, datetime.datetime):
@@ -53,28 +61,36 @@ def encode(cls, data):
).encode("utf-8")
-class OpenPypeRestApiResource(RestApiResource):
+class RestApiResource(JsonApiResource):
+ """Resource carrying needed info and Avalon DB connection for publish."""
+ def __init__(self, server_manager, executable, upload_dir,
+ studio_task_queue=None):
+ self.server_manager = server_manager
+ self.upload_dir = upload_dir
+ self.executable = executable
+
+ if studio_task_queue is None:
+ studio_task_queue = collections.deque().dequeu
+ self.studio_task_queue = studio_task_queue
+
+
+class WebpublishRestApiResource(JsonApiResource):
"""Resource carrying OP DB connection for storing batch info into DB."""
- def __init__(self, ):
- mongo_client = OpenPypeMongoConnection.get_mongo_client()
- database_name = os.environ["OPENPYPE_DATABASE_NAME"]
- self.dbcon = mongo_client[database_name]["webpublishes"]
+
+ def __init__(self):
+ self.dbcon = get_webpublish_conn()
-class WebpublisherProjectsEndpoint(_RestApiEndpoint):
+class ProjectsEndpoint(ResourceRestApiEndpoint):
"""Returns list of dict with project info (id, name)."""
async def get(self) -> Response:
output = []
- for project_name in self.dbcon.database.collection_names():
- project_doc = self.dbcon.database[project_name].find_one({
- "type": "project"
- })
- if project_doc:
- ret_val = {
- "id": project_doc["_id"],
- "name": project_doc["name"]
- }
- output.append(ret_val)
+ for project_doc in get_projects():
+ ret_val = {
+ "id": project_doc["_id"],
+ "name": project_doc["name"]
+ }
+ output.append(ret_val)
return Response(
status=200,
body=self.resource.encode(output),
@@ -82,7 +98,7 @@ async def get(self) -> Response:
)
-class WebpublisherHiearchyEndpoint(_RestApiEndpoint):
+class HiearchyEndpoint(ResourceRestApiEndpoint):
"""Returns dictionary with context tree from assets."""
async def get(self, project_name) -> Response:
query_projection = {
@@ -94,10 +110,7 @@ async def get(self, project_name) -> Response:
"type": 1,
}
- asset_docs = self.dbcon.database[project_name].find(
- {"type": "asset"},
- query_projection
- )
+ asset_docs = get_assets(project_name, fields=query_projection.keys())
asset_docs_by_id = {
asset_doc["_id"]: asset_doc
for asset_doc in asset_docs
@@ -181,7 +194,7 @@ def __init__(self, node_type, name):
self["attributes"] = {}
-class WebpublisherBatchPublishEndpoint(_RestApiEndpoint):
+class BatchPublishEndpoint(WebpublishApiEndpoint):
"""Triggers headless publishing of batch."""
async def post(self, request) -> Response:
# Validate existence of openpype executable
@@ -190,7 +203,7 @@ async def post(self, request) -> Response:
msg = "Non existent OpenPype executable {}".format(openpype_app)
raise RuntimeError(msg)
- log.info("WebpublisherBatchPublishEndpoint called")
+ log.info("BatchPublishEndpoint called")
content = await request.json()
# Each filter have extensions which are checked on first task item
@@ -201,7 +214,7 @@ async def post(self, request) -> Response:
# TVPaint filter
{
"extensions": [".tvpp"],
- "command": "remotepublish",
+ "command": "publish",
"arguments": {
"targets": ["tvpaint_worker"]
},
@@ -210,13 +223,13 @@ async def post(self, request) -> Response:
# Photoshop filter
{
"extensions": [".psd", ".psb"],
- "command": "remotepublishfromapp",
+ "command": "publishfromapp",
"arguments": {
- # Command 'remotepublishfromapp' requires --host argument
+ # Command 'publishfromapp' requires --host argument
"host": "photoshop",
# Make sure targets are set to None for cases that default
# would change
- # - targets argument is not used in 'remotepublishfromapp'
+ # - targets argument is not used in 'publishfromapp'
"targets": ["remotepublish"]
},
# does publish need to be handled by a queue, eg. only
@@ -228,7 +241,7 @@ async def post(self, request) -> Response:
batch_dir = os.path.join(self.resource.upload_dir, content["batch"])
# Default command and arguments
- command = "remotepublish"
+ command = "publish"
add_args = {
# All commands need 'project' and 'user'
"project": content["project_name"],
@@ -259,6 +272,8 @@ async def post(self, request) -> Response:
args = [
openpype_app,
+ "module",
+ WebpublisherAddon.name,
command,
batch_dir
]
@@ -286,7 +301,7 @@ async def post(self, request) -> Response:
)
-class WebpublisherTaskPublishEndpoint(_RestApiEndpoint):
+class TaskPublishEndpoint(WebpublishApiEndpoint):
"""Prepared endpoint triggered after each task - for future development."""
async def post(self, request) -> Response:
return Response(
@@ -296,31 +311,55 @@ async def post(self, request) -> Response:
)
-class BatchStatusEndpoint(_RestApiEndpoint):
- """Returns dict with info for batch_id."""
+class BatchStatusEndpoint(WebpublishApiEndpoint):
+ """Returns dict with info for batch_id.
+
+ Uses 'WebpublishRestApiResource'.
+ """
+
async def get(self, batch_id) -> Response:
output = self.dbcon.find_one({"batch_id": batch_id})
+ if output:
+ status = 200
+ else:
+ output = {"msg": "Batch id {} not found".format(batch_id),
+ "status": "queued",
+ "progress": 0}
+ status = 404
+ body = self.resource.encode(output)
return Response(
- status=200,
- body=self.resource.encode(output),
+ status=status,
+ body=body,
content_type="application/json"
)
-class PublishesStatusEndpoint(_RestApiEndpoint):
- """Returns list of dict with batch info for user (email address)."""
+class UserReportEndpoint(WebpublishApiEndpoint):
+ """Returns list of dict with batch info for user (email address).
+
+ Uses 'WebpublishRestApiResource'.
+ """
+
async def get(self, user) -> Response:
- output = list(self.dbcon.find({"user": user}))
+ output = list(self.dbcon.find({"user": user},
+ projection={"log": False}))
+
+ if output:
+ status = 200
+ else:
+ output = {"msg": "User {} not found".format(user)}
+ status = 404
+ body = self.resource.encode(output)
return Response(
- status=200,
- body=self.resource.encode(output),
+ status=status,
+ body=body,
content_type="application/json"
)
-class ConfiguredExtensionsEndpoint(_RestApiEndpoint):
+class ConfiguredExtensionsEndpoint(WebpublishApiEndpoint):
"""Returns dict of extensions which have mapping to family.
Returns:
@@ -335,19 +374,55 @@ async def get(self, project_name=None) -> Response:
configured = {
"file_exts": set(),
"sequence_exts": set(),
- # workfiles that could have "Studio Procesing" hardcoded for now
+ # workfiles that could have "Studio Processing" hardcoded for now
"studio_exts": set(["psd", "psb", "tvpp", "tvp"])
}
collect_conf = sett["webpublisher"]["publish"]["CollectPublishedFiles"]
- for _, mapping in collect_conf.get("task_type_to_family", {}).items():
- for _family, config in mapping.items():
- if config["is_sequence"]:
- configured["sequence_exts"].update(config["extensions"])
- else:
- configured["file_exts"].update(config["extensions"])
+ configs = collect_conf.get("task_type_to_family", [])
+ mappings = []
+ for _, conf_mappings in configs.items():
+ if isinstance(conf_mappings, dict):
+ conf_mappings = conf_mappings.values()
+ for conf_mapping in conf_mappings:
+ mappings.append(conf_mapping)
+
+ for mapping in mappings:
+ if mapping["is_sequence"]:
+ configured["sequence_exts"].update(mapping["extensions"])
+ else:
+ configured["file_exts"].update(mapping["extensions"])
return Response(
status=200,
body=self.resource.encode(dict(configured)),
content_type="application/json"
)
+
+
+class BatchReprocessEndpoint(WebpublishApiEndpoint):
+ """Marks latest 'batch_id' for reprocessing, returns 404 if not found.
+
+ Uses 'WebpublishRestApiResource'.
+ """
+
+ async def post(self, batch_id) -> Response:
+ batches = self.dbcon.find({"batch_id": batch_id,
+ "status": ERROR_STATUS}).sort("_id", -1)
+
+ if batches:
+ self.dbcon.update_one(
+ {"_id": batches[0]["_id"]},
+ {"$set": {"status": REPROCESS_STATUS}}
+ )
+ output = [{"msg": "Batch id {} set to reprocess".format(batch_id)}]
+ status = 200
+ else:
+ output = [{"msg": "Batch id {} not found".format(batch_id)}]
+ status = 404
+ body = self.resource.encode(output)
+
+ return Response(
+ status=status,
+ body=body,
+ content_type="application/json"
+ )
diff --git a/openpype/hosts/webpublisher/webserver_service/webserver.py b/openpype/hosts/webpublisher/webserver_service/webserver.py
new file mode 100644
index 00000000000..093b53d9d33
--- /dev/null
+++ b/openpype/hosts/webpublisher/webserver_service/webserver.py
@@ -0,0 +1,182 @@
+import collections
+import time
+import os
+from datetime import datetime
+import requests
+import json
+import subprocess
+
+from openpype.client import OpenPypeMongoConnection
+from openpype.modules import ModulesManager
+from openpype.lib import Logger
+
+from openpype_modules.webpublisher.lib import (
+ ERROR_STATUS,
+ REPROCESS_STATUS,
+ SENT_REPROCESSING_STATUS
+)
+
+from .webpublish_routes import (
+ RestApiResource,
+ WebpublishRestApiResource,
+ HiearchyEndpoint,
+ ProjectsEndpoint,
+ ConfiguredExtensionsEndpoint,
+ BatchPublishEndpoint,
+ BatchReprocessEndpoint,
+ BatchStatusEndpoint,
+ TaskPublishEndpoint,
+ UserReportEndpoint
+)
+
+log = Logger.get_logger("webserver_gui")
+
+
+def run_webserver(executable, upload_dir, host=None, port=None):
+ """Runs webserver in command line, adds routes."""
+
+ if not host:
+ host = "localhost"
+ if not port:
+ port = 8079
+
+ manager = ModulesManager()
+ webserver_module = manager.modules_by_name["webserver"]
+
+ server_manager = webserver_module.create_new_server_manager(port, host)
+ webserver_url = server_manager.url
+ # queue for remotepublishfromapp tasks
+ studio_task_queue = collections.deque()
+
+ resource = RestApiResource(server_manager,
+ upload_dir=upload_dir,
+ executable=executable,
+ studio_task_queue=studio_task_queue)
+ projects_endpoint = ProjectsEndpoint(resource)
+ server_manager.add_route(
+ "GET",
+ "/api/projects",
+ projects_endpoint.dispatch
+ )
+
+ hiearchy_endpoint = HiearchyEndpoint(resource)
+ server_manager.add_route(
+ "GET",
+ "/api/hierarchy/{project_name}",
+ hiearchy_endpoint.dispatch
+ )
+
+ configured_ext_endpoint = ConfiguredExtensionsEndpoint(resource)
+ server_manager.add_route(
+ "GET",
+ "/api/webpublish/configured_ext/{project_name}",
+ configured_ext_endpoint.dispatch
+ )
+
+ # triggers publish
+ webpublisher_task_publish_endpoint = BatchPublishEndpoint(resource)
+ server_manager.add_route(
+ "POST",
+ "/api/webpublish/batch",
+ webpublisher_task_publish_endpoint.dispatch
+ )
+
+ webpublisher_batch_publish_endpoint = TaskPublishEndpoint(resource)
+ server_manager.add_route(
+ "POST",
+ "/api/webpublish/task",
+ webpublisher_batch_publish_endpoint.dispatch
+ )
+
+ # reporting
+ webpublish_resource = WebpublishRestApiResource()
+ batch_status_endpoint = BatchStatusEndpoint(webpublish_resource)
+ server_manager.add_route(
+ "GET",
+ "/api/batch_status/{batch_id}",
+ batch_status_endpoint.dispatch
+ )
+
+ user_status_endpoint = UserReportEndpoint(webpublish_resource)
+ server_manager.add_route(
+ "GET",
+ "/api/publishes/{user}",
+ user_status_endpoint.dispatch
+ )
+
+ batch_reprocess_endpoint = BatchReprocessEndpoint(webpublish_resource)
+ server_manager.add_route(
+ "POST",
+ "/api/webpublish/reprocess/{batch_id}",
+ batch_reprocess_endpoint.dispatch
+ )
+
+ server_manager.start_server()
+ last_reprocessed = time.time()
+ while True:
+ if time.time() - last_reprocessed > 20:
+ reprocess_failed(upload_dir, webserver_url)
+ last_reprocessed = time.time()
+ if studio_task_queue:
+ args = studio_task_queue.popleft()
+ subprocess.call(args) # blocking call
+
+ time.sleep(1.0)
+
+
+def reprocess_failed(upload_dir, webserver_url):
+ # log.info("check_reprocesable_records")
+ mongo_client = OpenPypeMongoConnection.get_mongo_client()
+ database_name = os.environ["OPENPYPE_DATABASE_NAME"]
+ dbcon = mongo_client[database_name]["webpublishes"]
+
+ results = dbcon.find({"status": REPROCESS_STATUS})
+ reprocessed_batches = set()
+ for batch in results:
+ if batch["batch_id"] in reprocessed_batches:
+ continue
+
+ batch_url = os.path.join(upload_dir,
+ batch["batch_id"],
+ "manifest.json")
+ log.info("batch:: {} {}".format(os.path.exists(batch_url), batch_url))
+ if not os.path.exists(batch_url):
+ msg = "Manifest {} not found".format(batch_url)
+ print(msg)
+ dbcon.update_one(
+ {"_id": batch["_id"]},
+ {"$set":
+ {
+ "finish_date": datetime.now(),
+ "status": ERROR_STATUS,
+ "progress": 100,
+ "log": batch.get("log") + msg
+ }}
+ )
+ continue
+ server_url = "{}/api/webpublish/batch".format(webserver_url)
+
+ with open(batch_url) as f:
+ data = json.loads(f.read())
+
+ dbcon.update_many(
+ {
+ "batch_id": batch["batch_id"],
+ "status": {"$in": [ERROR_STATUS, REPROCESS_STATUS]}
+ },
+ {
+ "$set": {
+ "finish_date": datetime.now(),
+ "status": SENT_REPROCESSING_STATUS,
+ "progress": 100
+ }
+ }
+ )
+
+ try:
+ r = requests.post(server_url, json=data)
+ log.info("response{}".format(r))
+ except Exception:
+ log.info("exception", exc_info=True)
+
+ reprocessed_batches.add(batch["batch_id"])
diff --git a/openpype/hosts/webpublisher/webserver_service/webserver_cli.py b/openpype/hosts/webpublisher/webserver_service/webserver_cli.py
deleted file mode 100644
index c96ad8e110a..00000000000
--- a/openpype/hosts/webpublisher/webserver_service/webserver_cli.py
+++ /dev/null
@@ -1,158 +0,0 @@
-import collections
-import time
-import os
-from datetime import datetime
-import requests
-import json
-import subprocess
-
-from openpype.lib import PypeLogger
-
-from .webpublish_routes import (
- RestApiResource,
- OpenPypeRestApiResource,
- WebpublisherBatchPublishEndpoint,
- WebpublisherTaskPublishEndpoint,
- WebpublisherHiearchyEndpoint,
- WebpublisherProjectsEndpoint,
- BatchStatusEndpoint,
- PublishesStatusEndpoint,
- ConfiguredExtensionsEndpoint
-)
-
-
-log = PypeLogger().get_logger("webserver_gui")
-
-
-def run_webserver(*args, **kwargs):
- """Runs webserver in command line, adds routes."""
- from openpype.modules import ModulesManager
-
- manager = ModulesManager()
- webserver_module = manager.modules_by_name["webserver"]
- host = kwargs.get("host") or "localhost"
- port = kwargs.get("port") or 8079
- server_manager = webserver_module.create_new_server_manager(port, host)
- webserver_url = server_manager.url
- # queue for remotepublishfromapp tasks
- studio_task_queue = collections.deque()
-
- resource = RestApiResource(server_manager,
- upload_dir=kwargs["upload_dir"],
- executable=kwargs["executable"],
- studio_task_queue=studio_task_queue)
- projects_endpoint = WebpublisherProjectsEndpoint(resource)
- server_manager.add_route(
- "GET",
- "/api/projects",
- projects_endpoint.dispatch
- )
-
- hiearchy_endpoint = WebpublisherHiearchyEndpoint(resource)
- server_manager.add_route(
- "GET",
- "/api/hierarchy/{project_name}",
- hiearchy_endpoint.dispatch
- )
-
- configured_ext_endpoint = ConfiguredExtensionsEndpoint(resource)
- server_manager.add_route(
- "GET",
- "/api/webpublish/configured_ext/{project_name}",
- configured_ext_endpoint.dispatch
- )
-
- # triggers publish
- webpublisher_task_publish_endpoint = \
- WebpublisherBatchPublishEndpoint(resource)
- server_manager.add_route(
- "POST",
- "/api/webpublish/batch",
- webpublisher_task_publish_endpoint.dispatch
- )
-
- webpublisher_batch_publish_endpoint = \
- WebpublisherTaskPublishEndpoint(resource)
- server_manager.add_route(
- "POST",
- "/api/webpublish/task",
- webpublisher_batch_publish_endpoint.dispatch
- )
-
- # reporting
- openpype_resource = OpenPypeRestApiResource()
- batch_status_endpoint = BatchStatusEndpoint(openpype_resource)
- server_manager.add_route(
- "GET",
- "/api/batch_status/{batch_id}",
- batch_status_endpoint.dispatch
- )
-
- user_status_endpoint = PublishesStatusEndpoint(openpype_resource)
- server_manager.add_route(
- "GET",
- "/api/publishes/{user}",
- user_status_endpoint.dispatch
- )
-
- server_manager.start_server()
- last_reprocessed = time.time()
- while True:
- if time.time() - last_reprocessed > 20:
- reprocess_failed(kwargs["upload_dir"], webserver_url)
- last_reprocessed = time.time()
- if studio_task_queue:
- args = studio_task_queue.popleft()
- subprocess.call(args) # blocking call
-
- time.sleep(1.0)
-
-
-def reprocess_failed(upload_dir, webserver_url):
- # log.info("check_reprocesable_records")
- from openpype.lib import OpenPypeMongoConnection
-
- mongo_client = OpenPypeMongoConnection.get_mongo_client()
- database_name = os.environ["OPENPYPE_DATABASE_NAME"]
- dbcon = mongo_client[database_name]["webpublishes"]
-
- results = dbcon.find({"status": "reprocess"})
- for batch in results:
- batch_url = os.path.join(upload_dir,
- batch["batch_id"],
- "manifest.json")
- log.info("batch:: {} {}".format(os.path.exists(batch_url), batch_url))
- if not os.path.exists(batch_url):
- msg = "Manifest {} not found".format(batch_url)
- print(msg)
- dbcon.update_one(
- {"_id": batch["_id"]},
- {"$set":
- {
- "finish_date": datetime.now(),
- "status": "error",
- "progress": 1,
- "log": batch.get("log") + msg
- }}
- )
- continue
- server_url = "{}/api/webpublish/batch".format(webserver_url)
-
- with open(batch_url) as f:
- data = json.loads(f.read())
-
- try:
- r = requests.post(server_url, json=data)
- log.info("response{}".format(r))
- except Exception:
- log.info("exception", exc_info=True)
-
- dbcon.update_one(
- {"_id": batch["_id"]},
- {"$set":
- {
- "finish_date": datetime.now(),
- "status": "sent_for_reprocessing",
- "progress": 1
- }}
- )
diff --git a/openpype/lib/__init__.py b/openpype/lib/__init__.py
index c99e3bc28d7..17aafc3e8ba 100644
--- a/openpype/lib/__init__.py
+++ b/openpype/lib/__init__.py
@@ -16,39 +16,80 @@
site.addsitedir(python_version_dir)
+from .events import (
+ emit_event,
+ register_event_callback
+)
+
+from .vendor_bin_utils import (
+ find_executable,
+ get_vendor_bin_path,
+ get_oiio_tools_path,
+ get_ffmpeg_tool_path,
+ is_oiio_supported
+)
+
+from .attribute_definitions import (
+ AbtractAttrDef,
+
+ UIDef,
+ UISeparatorDef,
+ UILabelDef,
+
+ UnknownDef,
+ NumberDef,
+ TextDef,
+ EnumDef,
+ BoolDef,
+ FileDef,
+ FileDefItem,
+)
+
from .env_tools import (
env_value_to_bool,
get_paths_from_environ,
- get_global_environments
)
from .terminal import Terminal
from .execute import (
+ get_openpype_execute_args,
get_pype_execute_args,
+ get_linux_launcher_args,
execute,
run_subprocess,
+ run_detached_process,
+ run_openpype_process,
+ clean_envs_for_openpype_process,
path_to_subprocess_arg,
CREATE_NO_WINDOW
)
-from .log import PypeLogger, timeit
+from .log import (
+ Logger,
+ PypeLogger,
+)
+
+from .path_templates import (
+ merge_dict,
+ TemplateMissingKey,
+ TemplateUnsolved,
+ StringTemplate,
+ TemplatesDict,
+ FormatObject,
+)
+
from .mongo import (
get_default_components,
validate_mongo_connection,
OpenPypeMongoConnection
)
from .anatomy import (
- merge_dict,
Anatomy
)
-from .config import get_datetime_data
-
-from .vendor_bin_utils import (
- get_vendor_bin_path,
- get_oiio_tools_path,
- get_ffmpeg_tool_path,
- ffprobe_streams,
- is_oiio_supported
+from .dateutils import (
+ get_datetime_data,
+ get_timestamp,
+ get_formatted_current_time
)
from .python_module_tools import (
@@ -67,7 +108,14 @@
from .transcoding import (
get_transcode_temp_directory,
should_convert_for_ffmpeg,
- convert_for_ffmpeg
+ convert_for_ffmpeg,
+ convert_input_paths_for_ffmpeg,
+ get_ffprobe_data,
+ get_ffprobe_streams,
+ get_ffmpeg_codec_args,
+ get_ffmpeg_format_args,
+ convert_ffprobe_fps_value,
+ convert_ffprobe_fps_to_float,
)
from .avalon_context import (
CURRENT_DOC_SCHEMAS,
@@ -77,15 +125,17 @@
is_latest,
any_outdated,
get_asset,
- get_hierarchy,
get_linked_assets,
get_latest_version,
+ get_system_general_anatomy_data,
get_workfile_template_key,
get_workfile_template_key_from_context,
get_workdir_data,
get_workdir,
get_workdir_with_workdir_data,
+ get_last_workfile_with_version,
+ get_last_workfile,
create_workfile_doc,
save_workfile_data_to_doc,
@@ -125,7 +175,7 @@
PostLaunchHook,
EnvironmentPrepData,
- prepare_host_environments,
+ prepare_app_environments,
prepare_context_environments,
get_app_environments_for_context,
apply_project_environments_value
@@ -139,22 +189,25 @@
filter_pyblish_plugins,
set_plugin_attributes_from_settings,
source_hash,
- get_unique_layer_name,
- get_background_layers,
)
from .path_tools import (
+ format_file_size,
+ collect_frames,
+ create_hard_link,
version_up,
get_version_from_path,
get_last_version_from_path,
create_project_folders,
- get_project_basic_paths
+ create_workdir_extra_folders,
+ get_project_basic_paths,
)
from .editorial import (
is_overlapping_otio_ranges,
otio_range_to_frame_range,
otio_range_with_handles,
+ get_media_range_with_retimes,
convert_to_padded_path,
trim_media_range,
range_from_frames,
@@ -163,30 +216,63 @@
make_sequence_collection
)
-from .pype_info import (
+from .openpype_version import (
+ op_version_control_available,
get_openpype_version,
- get_build_version
+ get_build_version,
+ get_expected_version,
+ is_running_from_build,
+ is_running_staging,
+ is_current_version_studio_latest,
+ is_current_version_higher_than_expected
+)
+
+
+from .connections import (
+ requests_get,
+ requests_post
)
terminal = Terminal
__all__ = [
+ "emit_event",
+ "register_event_callback",
+
+ "find_executable",
+ "get_openpype_execute_args",
"get_pype_execute_args",
+ "get_linux_launcher_args",
"execute",
"run_subprocess",
+ "run_detached_process",
+ "run_openpype_process",
+ "clean_envs_for_openpype_process",
"path_to_subprocess_arg",
"CREATE_NO_WINDOW",
"env_value_to_bool",
"get_paths_from_environ",
- "get_global_environments",
"get_vendor_bin_path",
"get_oiio_tools_path",
"get_ffmpeg_tool_path",
- "ffprobe_streams",
"is_oiio_supported",
+ "AbtractAttrDef",
+
+ "UIDef",
+ "UISeparatorDef",
+ "UILabelDef",
+
+ "UnknownDef",
+ "NumberDef",
+ "TextDef",
+ "EnumDef",
+ "BoolDef",
+ "FileDef",
+ "FileDefItem",
+
"import_filepath",
"modules_from_path",
"recursive_bases_from_class",
@@ -196,6 +282,13 @@
"get_transcode_temp_directory",
"should_convert_for_ffmpeg",
"convert_for_ffmpeg",
+ "convert_input_paths_for_ffmpeg",
+ "get_ffprobe_data",
+ "get_ffprobe_streams",
+ "get_ffmpeg_codec_args",
+ "get_ffmpeg_format_args",
+ "convert_ffprobe_fps_value",
+ "convert_ffprobe_fps_to_float",
"CURRENT_DOC_SCHEMAS",
"PROJECT_NAME_ALLOWED_SYMBOLS",
@@ -204,15 +297,17 @@
"is_latest",
"any_outdated",
"get_asset",
- "get_hierarchy",
"get_linked_assets",
"get_latest_version",
+ "get_system_general_anatomy_data",
"get_workfile_template_key",
"get_workfile_template_key_from_context",
"get_workdir_data",
"get_workdir",
"get_workdir_with_workdir_data",
+ "get_last_workfile_with_version",
+ "get_last_workfile",
"create_workfile_doc",
"save_workfile_data_to_doc",
@@ -244,7 +339,7 @@
"PreLaunchHook",
"PostLaunchHook",
"EnvironmentPrepData",
- "prepare_host_environments",
+ "prepare_app_environments",
"prepare_context_environments",
"get_app_environments_for_context",
"apply_project_environments_value",
@@ -259,39 +354,57 @@
"filter_pyblish_plugins",
"set_plugin_attributes_from_settings",
"source_hash",
- "get_unique_layer_name",
- "get_background_layers",
+ "format_file_size",
+ "collect_frames",
+ "create_hard_link",
"version_up",
"get_version_from_path",
"get_last_version_from_path",
+ "merge_dict",
+ "TemplateMissingKey",
+ "TemplateUnsolved",
+ "StringTemplate",
+ "TemplatesDict",
+ "FormatObject",
+
"terminal",
- "merge_dict",
"Anatomy",
"get_datetime_data",
+ "get_formatted_current_time",
+ "Logger",
"PypeLogger",
+
"get_default_components",
"validate_mongo_connection",
"OpenPypeMongoConnection",
- "timeit",
-
"is_overlapping_otio_ranges",
"otio_range_with_handles",
"convert_to_padded_path",
"otio_range_to_frame_range",
+ "get_media_range_with_retimes",
"trim_media_range",
"range_from_frames",
"frames_to_secons",
"frames_to_timecode",
"make_sequence_collection",
"create_project_folders",
+ "create_workdir_extra_folders",
"get_project_basic_paths",
+ "op_version_control_available",
"get_openpype_version",
"get_build_version",
+ "get_expected_version",
+ "is_running_from_build",
+ "is_running_staging",
+ "is_current_version_studio_latest",
+
+ "requests_get",
+ "requests_post"
]
diff --git a/openpype/lib/abstract_collect_render.py b/openpype/lib/abstract_collect_render.py
index 2ac0fe434d1..e4ff87aa0fb 100644
--- a/openpype/lib/abstract_collect_render.py
+++ b/openpype/lib/abstract_collect_render.py
@@ -1,264 +1,33 @@
# -*- coding: utf-8 -*-
-"""Collect render template.
+"""Content was moved to 'openpype.pipeline.publish.abstract_collect_render'.
-TODO: use @dataclass when times come.
+Please change your imports as soon as possible.
+File will be probably removed in OpenPype 3.14.*
"""
-from abc import abstractmethod
-import attr
-import six
+import warnings
+from openpype.pipeline.publish import AbstractCollectRender, RenderInstance
-from avalon import api
-import pyblish.api
-from .abstract_metaplugins import AbstractMetaContextPlugin
+class CollectRenderDeprecated(DeprecationWarning):
+ pass
-@attr.s
-class RenderInstance(object):
- """Data collected by collectors.
+warnings.simplefilter("always", CollectRenderDeprecated)
+warnings.warn(
+ (
+ "Content of 'abstract_collect_render' was moved."
+ "\nUsing deprecated source of 'abstract_collect_render'. Content was"
+ " move to 'openpype.pipeline.publish.abstract_collect_render'."
+ " Please change your imports as soon as possible."
+ ),
+ category=CollectRenderDeprecated,
+ stacklevel=4
+)
- This data class later on passed to collected instances.
- Those attributes are required later on.
- """
-
- # metadata
- version = attr.ib() # instance version
- time = attr.ib() # time of instance creation (avalon.api.time())
- source = attr.ib() # path to source scene file
- label = attr.ib() # label to show in GUI
- subset = attr.ib() # subset name
- asset = attr.ib() # asset name (AVALON_ASSET)
- attachTo = attr.ib() # subset name to attach render to
- setMembers = attr.ib() # list of nodes/members producing render output
- publish = attr.ib() # bool, True to publish instance
- name = attr.ib() # instance name
-
- # format settings
- resolutionWidth = attr.ib() # resolution width (1920)
- resolutionHeight = attr.ib() # resolution height (1080)
- pixelAspect = attr.ib() # pixel aspect (1.0)
-
- # time settings
- frameStart = attr.ib() # start frame
- frameEnd = attr.ib() # start end
- frameStep = attr.ib() # frame step
-
- handleStart = attr.ib(default=None) # start frame
- handleEnd = attr.ib(default=None) # start frame
-
- # for softwares (like Harmony) where frame range cannot be set by DB
- # handles need to be propagated if exist
- ignoreFrameHandleCheck = attr.ib(default=False)
-
- # --------------------
- # With default values
- # metadata
- renderer = attr.ib(default="") # renderer - can be used in Deadline
- review = attr.ib(default=False) # genereate review from instance (bool)
- priority = attr.ib(default=50) # job priority on farm
-
- family = attr.ib(default="renderlayer")
- families = attr.ib(default=["renderlayer"]) # list of families
-
- # format settings
- multipartExr = attr.ib(default=False) # flag for multipart exrs
- convertToScanline = attr.ib(default=False) # flag for exr conversion
-
- tileRendering = attr.ib(default=False) # bool: treat render as tiles
- tilesX = attr.ib(default=0) # number of tiles in X
- tilesY = attr.ib(default=0) # number of tiles in Y
-
- # submit_publish_job
- toBeRenderedOn = attr.ib(default=None)
- deadlineSubmissionJob = attr.ib(default=None)
- anatomyData = attr.ib(default=None)
- outputDir = attr.ib(default=None)
-
- @frameStart.validator
- def check_frame_start(self, _, value):
- """Validate if frame start is not larger then end."""
- if value > self.frameEnd:
- raise ValueError("frameStart must be smaller "
- "or equal then frameEnd")
-
- @frameEnd.validator
- def check_frame_end(self, _, value):
- """Validate if frame end is not less then start."""
- if value < self.frameStart:
- raise ValueError("frameEnd must be smaller "
- "or equal then frameStart")
-
- @tilesX.validator
- def check_tiles_x(self, _, value):
- """Validate if tile x isn't less then 1."""
- if not self.tileRendering:
- return
- if value < 1:
- raise ValueError("tile X size cannot be less then 1")
-
- if value == 1 and self.tilesY == 1:
- raise ValueError("both tiles X a Y sizes are set to 1")
-
- @tilesY.validator
- def check_tiles_y(self, _, value):
- """Validate if tile y isn't less then 1."""
- if not self.tileRendering:
- return
- if value < 1:
- raise ValueError("tile Y size cannot be less then 1")
-
- if value == 1 and self.tilesX == 1:
- raise ValueError("both tiles X a Y sizes are set to 1")
-
-
-@six.add_metaclass(AbstractMetaContextPlugin)
-class AbstractCollectRender(pyblish.api.ContextPlugin):
- """Gather all publishable render layers from renderSetup."""
-
- order = pyblish.api.CollectorOrder + 0.01
- label = "Collect Render"
- sync_workfile_version = False
-
- def __init__(self, *args, **kwargs):
- """Constructor."""
- super(AbstractCollectRender, self).__init__(*args, **kwargs)
- self._file_path = None
- self._asset = api.Session["AVALON_ASSET"]
- self._context = None
-
- def process(self, context):
- """Entry point to collector."""
- self._context = context
- for instance in context:
- # make sure workfile instance publishing is enabled
- try:
- if "workfile" in instance.data["families"]:
- instance.data["publish"] = True
- if "renderFarm" in instance.data["families"]:
- instance.data["remove"] = True
- except KeyError:
- # be tolerant if 'families' is missing.
- pass
-
- self._file_path = context.data["currentFile"].replace("\\", "/")
-
- render_instances = self.get_instances(context)
- for render_instance in render_instances:
- exp_files = self.get_expected_files(render_instance)
- assert exp_files, "no file names were generated, this is bug"
-
- # if we want to attach render to subset, check if we have AOV's
- # in expectedFiles. If so, raise error as we cannot attach AOV
- # (considered to be subset on its own) to another subset
- if render_instance.attachTo:
- assert isinstance(exp_files, list), (
- "attaching multiple AOVs or renderable cameras to "
- "subset is not supported"
- )
-
- frame_start_render = int(render_instance.frameStart)
- frame_end_render = int(render_instance.frameEnd)
- if (render_instance.ignoreFrameHandleCheck or
- int(context.data['frameStartHandle']) == frame_start_render
- and int(context.data['frameEndHandle']) == frame_end_render): # noqa: W503, E501
-
- handle_start = context.data['handleStart']
- handle_end = context.data['handleEnd']
- frame_start = context.data['frameStart']
- frame_end = context.data['frameEnd']
- frame_start_handle = context.data['frameStartHandle']
- frame_end_handle = context.data['frameEndHandle']
- else:
- handle_start = 0
- handle_end = 0
- frame_start = frame_start_render
- frame_end = frame_end_render
- frame_start_handle = frame_start_render
- frame_end_handle = frame_end_render
-
- data = {
- "handleStart": handle_start,
- "handleEnd": handle_end,
- "frameStart": frame_start,
- "frameEnd": frame_end,
- "frameStartHandle": frame_start_handle,
- "frameEndHandle": frame_end_handle,
- "byFrameStep": int(render_instance.frameStep),
-
- "author": context.data["user"],
- # Add source to allow tracing back to the scene from
- # which was submitted originally
- "expectedFiles": exp_files,
- }
- if self.sync_workfile_version:
- data["version"] = context.data["version"]
-
- # add additional data
- data = self.add_additional_data(data)
- render_instance_dict = attr.asdict(render_instance)
-
- instance = context.create_instance(render_instance.name)
- instance.data["label"] = render_instance.label
- instance.data.update(render_instance_dict)
- instance.data.update(data)
-
- self.post_collecting_action()
-
- @abstractmethod
- def get_instances(self, context):
- """Get all renderable instances and their data.
-
- Args:
- context (pyblish.api.Context): Context object.
-
- Returns:
- list of :class:`RenderInstance`: All collected renderable instances
- (like render layers, write nodes, etc.)
-
- """
- pass
-
- @abstractmethod
- def get_expected_files(self, render_instance):
- """Get list of expected files.
-
- Returns:
- list: expected files. This can be either simple list of files with
- their paths, or list of dictionaries, where key is name of AOV
- for example and value is list of files for that AOV.
-
- Example::
-
- ['/path/to/file.001.exr', '/path/to/file.002.exr']
-
- or as dictionary:
-
- [
- {
- "beauty": ['/path/to/beauty.001.exr', ...],
- "mask": ['/path/to/mask.001.exr']
- }
- ]
-
- """
- pass
-
- def add_additional_data(self, data):
- """Add additional data to collected instance.
-
- This can be overridden by host implementation to add custom
- additional data.
-
- """
- return data
-
- def post_collecting_action(self):
- """Execute some code after collection is done.
-
- This is useful for example for restoring current render layer.
-
- """
- pass
+__all__ = (
+ "AbstractCollectRender",
+ "RenderInstance"
+)
diff --git a/openpype/lib/abstract_expected_files.py b/openpype/lib/abstract_expected_files.py
index f9f3c17ef5d..f24d844fe57 100644
--- a/openpype/lib/abstract_expected_files.py
+++ b/openpype/lib/abstract_expected_files.py
@@ -1,53 +1,32 @@
# -*- coding: utf-8 -*-
-"""Abstract ExpectedFile class definition."""
-from abc import ABCMeta, abstractmethod
-import six
+"""Content was moved to 'openpype.pipeline.publish.abstract_expected_files'.
+Please change your imports as soon as possible.
-@six.add_metaclass(ABCMeta)
-class ExpectedFiles:
- """Class grouping functionality for all supported renderers.
+File will be probably removed in OpenPype 3.14.*
+"""
- Attributes:
- multipart (bool): Flag if multipart exrs are used.
+import warnings
+from openpype.pipeline.publish import ExpectedFiles
- """
- multipart = False
+class ExpectedFilesDeprecated(DeprecationWarning):
+ pass
- @abstractmethod
- def get(self, render_instance):
- """Get expected files for given renderer and render layer.
- This method should return dictionary of all files we are expecting
- to be rendered from the host. Usually `render_instance` corresponds
- to *render layer*. Result can be either flat list with the file
- paths or it can be list of dictionaries. Each key corresponds to
- for example AOV name or channel, etc.
+warnings.simplefilter("always", ExpectedFilesDeprecated)
+warnings.warn(
+ (
+ "Content of 'abstract_expected_files' was moved."
+ "\nUsing deprecated source of 'abstract_expected_files'. Content was"
+ " move to 'openpype.pipeline.publish.abstract_expected_files'."
+ " Please change your imports as soon as possible."
+ ),
+ category=ExpectedFilesDeprecated,
+ stacklevel=4
+)
- Example::
- ['/path/to/file.001.exr', '/path/to/file.002.exr']
-
- or as dictionary:
-
- [
- {
- "beauty": ['/path/to/beauty.001.exr', ...],
- "mask": ['/path/to/mask.001.exr']
- }
- ]
-
-
- Args:
- render_instance (:class:`RenderInstance`): Data passed from
- collector to determine files. This should be instance of
- :class:`abstract_collect_render.RenderInstance`
-
- Returns:
- list: Full paths to expected rendered files.
- list of dict: Path to expected rendered files categorized by
- AOVs, etc.
-
- """
- raise NotImplementedError()
+__all__ = (
+ "ExpectedFiles",
+)
diff --git a/openpype/lib/abstract_metaplugins.py b/openpype/lib/abstract_metaplugins.py
index f8163956ad8..346b5d86b3b 100644
--- a/openpype/lib/abstract_metaplugins.py
+++ b/openpype/lib/abstract_metaplugins.py
@@ -1,10 +1,35 @@
-from abc import ABCMeta
-from pyblish.plugin import MetaPlugin, ExplicitMetaPlugin
+"""Content was moved to 'openpype.pipeline.publish.publish_plugins'.
+Please change your imports as soon as possible.
-class AbstractMetaInstancePlugin(ABCMeta, MetaPlugin):
- pass
+File will be probably removed in OpenPype 3.14.*
+"""
+
+import warnings
+from openpype.pipeline.publish import (
+ AbstractMetaInstancePlugin,
+ AbstractMetaContextPlugin
+)
-class AbstractMetaContextPlugin(ABCMeta, ExplicitMetaPlugin):
+class MetaPluginsDeprecated(DeprecationWarning):
pass
+
+
+warnings.simplefilter("always", MetaPluginsDeprecated)
+warnings.warn(
+ (
+ "Content of 'abstract_metaplugins' was moved."
+ "\nUsing deprecated source of 'abstract_metaplugins'. Content was"
+ " moved to 'openpype.pipeline.publish.publish_plugins'."
+ " Please change your imports as soon as possible."
+ ),
+ category=MetaPluginsDeprecated,
+ stacklevel=4
+)
+
+
+__all__ = (
+ "AbstractMetaInstancePlugin",
+ "AbstractMetaContextPlugin",
+)
diff --git a/openpype/lib/abstract_submit_deadline.py b/openpype/lib/abstract_submit_deadline.py
deleted file mode 100644
index 5b6e1743e0a..00000000000
--- a/openpype/lib/abstract_submit_deadline.py
+++ /dev/null
@@ -1,634 +0,0 @@
-# -*- coding: utf-8 -*-
-"""Abstract package for submitting jobs to Deadline.
-
-It provides Deadline JobInfo data class.
-
-"""
-import os
-from abc import abstractmethod
-import platform
-import getpass
-from collections import OrderedDict
-
-import six
-import attr
-import requests
-
-import pyblish.api
-from .abstract_metaplugins import AbstractMetaInstancePlugin
-
-
-def requests_post(*args, **kwargs):
- """Wrap request post method.
-
- Disabling SSL certificate validation if ``DONT_VERIFY_SSL`` environment
- variable is found. This is useful when Deadline or Muster server are
- running with self-signed certificates and their certificate is not
- added to trusted certificates on client machines.
-
- Warning:
- Disabling SSL certificate validation is defeating one line
- of defense SSL is providing and it is not recommended.
-
- """
- if 'verify' not in kwargs:
- kwargs['verify'] = False if os.getenv("OPENPYPE_DONT_VERIFY_SSL",
- True) else True # noqa
- # add 10sec timeout before bailing out
- kwargs['timeout'] = 10
- return requests.post(*args, **kwargs)
-
-
-def requests_get(*args, **kwargs):
- """Wrap request get method.
-
- Disabling SSL certificate validation if ``DONT_VERIFY_SSL`` environment
- variable is found. This is useful when Deadline or Muster server are
- running with self-signed certificates and their certificate is not
- added to trusted certificates on client machines.
-
- Warning:
- Disabling SSL certificate validation is defeating one line
- of defense SSL is providing and it is not recommended.
-
- """
- if 'verify' not in kwargs:
- kwargs['verify'] = False if os.getenv("OPENPYPE_DONT_VERIFY_SSL",
- True) else True # noqa
- # add 10sec timeout before bailing out
- kwargs['timeout'] = 10
- return requests.get(*args, **kwargs)
-
-
-@attr.s
-class DeadlineJobInfo(object):
- """Mapping of all Deadline *JobInfo* attributes.
-
- This contains all JobInfo attributes plus their default values.
- Those attributes set to `None` shouldn't be posted to Deadline as
- the only required one is `Plugin`. Their default values used by Deadline
- are stated in
- comments.
-
- ..seealso:
- https://docs.thinkboxsoftware.com/products/deadline/10.1/1_User%20Manual/manual/manual-submission.html
-
- """
-
- # Required
- # ----------------------------------------------
- Plugin = attr.ib()
-
- # General
- Frames = attr.ib(default=None) # default: 0
- Name = attr.ib(default="Untitled")
- Comment = attr.ib(default=None) # default: empty
- Department = attr.ib(default=None) # default: empty
- BatchName = attr.ib(default=None) # default: empty
- UserName = attr.ib(default=getpass.getuser())
- MachineName = attr.ib(default=platform.node())
- Pool = attr.ib(default=None) # default: "none"
- SecondaryPool = attr.ib(default=None)
- Group = attr.ib(default=None) # default: "none"
- Priority = attr.ib(default=50)
- ChunkSize = attr.ib(default=1)
- ConcurrentTasks = attr.ib(default=1)
- LimitConcurrentTasksToNumberOfCpus = attr.ib(
- default=None) # default: "true"
- OnJobComplete = attr.ib(default="Nothing")
- SynchronizeAllAuxiliaryFiles = attr.ib(default=None) # default: false
- ForceReloadPlugin = attr.ib(default=None) # default: false
- Sequential = attr.ib(default=None) # default: false
- SuppressEvents = attr.ib(default=None) # default: false
- Protected = attr.ib(default=None) # default: false
- InitialStatus = attr.ib(default="Active")
- NetworkRoot = attr.ib(default=None)
-
- # Timeouts
- # ----------------------------------------------
- MinRenderTimeSeconds = attr.ib(default=None) # Default: 0
- MinRenderTimeMinutes = attr.ib(default=None) # Default: 0
- TaskTimeoutSeconds = attr.ib(default=None) # Default: 0
- TaskTimeoutMinutes = attr.ib(default=None) # Default: 0
- StartJobTimeoutSeconds = attr.ib(default=None) # Default: 0
- StartJobTimeoutMinutes = attr.ib(default=None) # Default: 0
- InitializePluginTimeoutSeconds = attr.ib(default=None) # Default: 0
- # can be one of
- OnTaskTimeout = attr.ib(default=None) # Default: Error
- EnableTimeoutsForScriptTasks = attr.ib(default=None) # Default: false
- EnableFrameTimeouts = attr.ib(default=None) # Default: false
- EnableAutoTimeout = attr.ib(default=None) # Default: false
-
- # Interruptible
- # ----------------------------------------------
- Interruptible = attr.ib(default=None) # Default: false
- InterruptiblePercentage = attr.ib(default=None)
- RemTimeThreshold = attr.ib(default=None)
-
- # Notifications
- # ----------------------------------------------
- # can be comma separated list of users
- NotificationTargets = attr.ib(default=None) # Default: blank
- ClearNotificationTargets = attr.ib(default=None) # Default: false
- # A comma separated list of additional email addresses
- NotificationEmails = attr.ib(default=None) # Default: blank
- OverrideNotificationMethod = attr.ib(default=None) # Default: false
- EmailNotification = attr.ib(default=None) # Default: false
- PopupNotification = attr.ib(default=None) # Default: false
- # String with `[EOL]` used for end of line
- NotificationNote = attr.ib(default=None) # Default: blank
-
- # Machine Limit
- # ----------------------------------------------
- MachineLimit = attr.ib(default=None) # Default: 0
- MachineLimitProgress = attr.ib(default=None) # Default: -1.0
- Whitelist = attr.ib(default=None) # Default: blank
- Blacklist = attr.ib(default=None) # Default: blank
-
- # Limits
- # ----------------------------------------------
- # comma separated list of limit groups
- LimitGroups = attr.ib(default=None) # Default: blank
-
- # Dependencies
- # ----------------------------------------------
- # comma separated list of job IDs
- JobDependencies = attr.ib(default=None) # Default: blank
- JobDependencyPercentage = attr.ib(default=None) # Default: -1
- IsFrameDependent = attr.ib(default=None) # Default: false
- FrameDependencyOffsetStart = attr.ib(default=None) # Default: 0
- FrameDependencyOffsetEnd = attr.ib(default=None) # Default: 0
- ResumeOnCompleteDependencies = attr.ib(default=None) # Default: true
- ResumeOnDeletedDependencies = attr.ib(default=None) # Default: false
- ResumeOnFailedDependencies = attr.ib(default=None) # Default: false
- # comma separated list of asset paths
- RequiredAssets = attr.ib(default=None) # Default: blank
- # comma separated list of script paths
- ScriptDependencies = attr.ib(default=None) # Default: blank
-
- # Failure Detection
- # ----------------------------------------------
- OverrideJobFailureDetection = attr.ib(default=None) # Default: false
- FailureDetectionJobErrors = attr.ib(default=None) # 0..x
- OverrideTaskFailureDetection = attr.ib(default=None) # Default: false
- FailureDetectionTaskErrors = attr.ib(default=None) # 0..x
- IgnoreBadJobDetection = attr.ib(default=None) # Default: false
- SendJobErrorWarning = attr.ib(default=None) # Default: false
-
- # Cleanup
- # ----------------------------------------------
- DeleteOnComplete = attr.ib(default=None) # Default: false
- ArchiveOnComplete = attr.ib(default=None) # Default: false
- OverrideAutoJobCleanup = attr.ib(default=None) # Default: false
- OverrideJobCleanup = attr.ib(default=None)
- JobCleanupDays = attr.ib(default=None) # Default: false
- #
- OverrideJobCleanupType = attr.ib(default=None)
-
- # Scheduling
- # ----------------------------------------------
- #
- ScheduledType = attr.ib(default=None) # Default: None
- #
- ScheduledStartDateTime = attr.ib(default=None)
- ScheduledDays = attr.ib(default=None) # Default: 1
- #
- JobDelay = attr.ib(default=None)
- # Time=
- Scheduled = attr.ib(default=None)
-
- # Scripts
- # ----------------------------------------------
- # all accept path to script
- PreJobScript = attr.ib(default=None) # Default: blank
- PostJobScript = attr.ib(default=None) # Default: blank
- PreTaskScript = attr.ib(default=None) # Default: blank
- PostTaskScript = attr.ib(default=None) # Default: blank
-
- # Event Opt-Ins
- # ----------------------------------------------
- # comma separated list of plugins
- EventOptIns = attr.ib(default=None) # Default: blank
-
- # Environment
- # ----------------------------------------------
- _environmentKeyValue = attr.ib(factory=list)
-
- @property
- def EnvironmentKeyValue(self): # noqa: N802
- """Return all environment key values formatted for Deadline.
-
- Returns:
- dict: as `{'EnvironmentKeyValue0', 'key=value'}`
-
- """
- out = {}
- for index, v in enumerate(self._environmentKeyValue):
- out["EnvironmentKeyValue{}".format(index)] = v
- return out
-
- @EnvironmentKeyValue.setter
- def EnvironmentKeyValue(self, val): # noqa: N802
- self._environmentKeyValue.append(val)
-
- IncludeEnvironment = attr.ib(default=None) # Default: false
- UseJobEnvironmentOnly = attr.ib(default=None) # Default: false
- CustomPluginDirectory = attr.ib(default=None) # Default: blank
-
- # Job Extra Info
- # ----------------------------------------------
- _extraInfos = attr.ib(factory=list)
- _extraInfoKeyValues = attr.ib(factory=list)
-
- @property
- def ExtraInfo(self): # noqa: N802
- """Return all ExtraInfo values formatted for Deadline.
-
- Returns:
- dict: as `{'ExtraInfo0': 'value'}`
-
- """
- out = {}
- for index, v in enumerate(self._extraInfos):
- out["ExtraInfo{}".format(index)] = v
- return out
-
- @ExtraInfo.setter
- def ExtraInfo(self, val): # noqa: N802
- self._extraInfos.append(val)
-
- @property
- def ExtraInfoKeyValue(self): # noqa: N802
- """Return all ExtraInfoKeyValue values formatted for Deadline.
-
- Returns:
- dict: as {'ExtraInfoKeyValue0': 'key=value'}`
-
- """
- out = {}
- for index, v in enumerate(self._extraInfoKeyValues):
- out["ExtraInfoKeyValue{}".format(index)] = v
- return out
-
- @ExtraInfoKeyValue.setter
- def ExtraInfoKeyValue(self, val): # noqa: N802
- self._extraInfoKeyValues.append(val)
-
- # Task Extra Info Names
- # ----------------------------------------------
- OverrideTaskExtraInfoNames = attr.ib(default=None) # Default: false
- _taskExtraInfos = attr.ib(factory=list)
-
- @property
- def TaskExtraInfoName(self): # noqa: N802
- """Return all TaskExtraInfoName values formatted for Deadline.
-
- Returns:
- dict: as `{'TaskExtraInfoName0': 'value'}`
-
- """
- out = {}
- for index, v in enumerate(self._taskExtraInfos):
- out["TaskExtraInfoName{}".format(index)] = v
- return out
-
- @TaskExtraInfoName.setter
- def TaskExtraInfoName(self, val): # noqa: N802
- self._taskExtraInfos.append(val)
-
- # Output
- # ----------------------------------------------
- _outputFilename = attr.ib(factory=list)
- _outputFilenameTile = attr.ib(factory=list)
- _outputDirectory = attr.ib(factory=list)
-
- @property
- def OutputFilename(self): # noqa: N802
- """Return all OutputFilename values formatted for Deadline.
-
- Returns:
- dict: as `{'OutputFilename0': 'filename'}`
-
- """
- out = {}
- for index, v in enumerate(self._outputFilename):
- out["OutputFilename{}".format(index)] = v
- return out
-
- @OutputFilename.setter
- def OutputFilename(self, val): # noqa: N802
- self._outputFilename.append(val)
-
- @property
- def OutputFilenameTile(self): # noqa: N802
- """Return all OutputFilename#Tile values formatted for Deadline.
-
- Returns:
- dict: as `{'OutputFilenme#Tile': 'tile'}`
-
- """
- out = {}
- for index, v in enumerate(self._outputFilenameTile):
- out["OutputFilename{}Tile".format(index)] = v
- return out
-
- @OutputFilenameTile.setter
- def OutputFilenameTile(self, val): # noqa: N802
- self._outputFilenameTile.append(val)
-
- @property
- def OutputDirectory(self): # noqa: N802
- """Return all OutputDirectory values formatted for Deadline.
-
- Returns:
- dict: as `{'OutputDirectory0': 'dir'}`
-
- """
- out = {}
- for index, v in enumerate(self._outputDirectory):
- out["OutputDirectory{}".format(index)] = v
- return out
-
- @OutputDirectory.setter
- def OutputDirectory(self, val): # noqa: N802
- self._outputDirectory.append(val)
-
- # Tile Job
- # ----------------------------------------------
- TileJob = attr.ib(default=None) # Default: false
- TileJobFrame = attr.ib(default=None) # Default: 0
- TileJobTilesInX = attr.ib(default=None) # Default: 0
- TileJobTilesInY = attr.ib(default=None) # Default: 0
- TileJobTileCount = attr.ib(default=None) # Default: 0
-
- # Maintenance Job
- # ----------------------------------------------
- MaintenanceJob = attr.ib(default=None) # Default: false
- MaintenanceJobStartFrame = attr.ib(default=None) # Default: 0
- MaintenanceJobEndFrame = attr.ib(default=None) # Default: 0
-
- def serialize(self):
- """Return all data serialized as dictionary.
-
- Returns:
- OrderedDict: all serialized data.
-
- """
- def filter_data(a, v):
- if a.name.startswith("_"):
- return False
- if v is None:
- return False
- return True
-
- serialized = attr.asdict(
- self, dict_factory=OrderedDict, filter=filter_data)
- serialized.update(self.EnvironmentKeyValue)
- serialized.update(self.ExtraInfo)
- serialized.update(self.ExtraInfoKeyValue)
- serialized.update(self.TaskExtraInfoName)
- serialized.update(self.OutputFilename)
- serialized.update(self.OutputFilenameTile)
- serialized.update(self.OutputDirectory)
- return serialized
-
-
-@six.add_metaclass(AbstractMetaInstancePlugin)
-class AbstractSubmitDeadline(pyblish.api.InstancePlugin):
- """Class abstracting access to Deadline."""
-
- label = "Submit to Deadline"
- order = pyblish.api.IntegratorOrder + 0.1
- use_published = True
- asset_dependencies = False
-
- def __init__(self, *args, **kwargs):
- super(AbstractSubmitDeadline, self).__init__(*args, **kwargs)
- self._instance = None
- self._deadline_url = None
- self.scene_path = None
- self.job_info = None
- self.plugin_info = None
- self.aux_files = None
-
- def process(self, instance):
- """Plugin entry point."""
- self._instance = instance
- context = instance.context
- self._deadline_url = context.data.get("defaultDeadline")
- self._deadline_url = instance.data.get(
- "deadlineUrl", self._deadline_url)
-
- assert self._deadline_url, "Requires Deadline Webservice URL"
-
- file_path = None
- if self.use_published:
- file_path = self.from_published_scene()
-
- # fallback if nothing was set
- if not file_path:
- self.log.warning("Falling back to workfile")
- file_path = context.data["currentFile"]
-
- self.scene_path = file_path
- self.log.info("Using {} for render/export.".format(file_path))
-
- self.job_info = self.get_job_info()
- self.plugin_info = self.get_plugin_info()
- self.aux_files = self.get_aux_files()
-
- self.process_submission()
-
- def process_submission(self):
- """Process data for submission.
-
- This takes Deadline JobInfo, PluginInfo, AuxFile, creates payload
- from them and submit it do Deadline.
-
- Returns:
- str: Deadline job ID
-
- """
- payload = self.assemble_payload()
- return self.submit(payload)
-
- @abstractmethod
- def get_job_info(self):
- """Return filled Deadline JobInfo.
-
- This is host/plugin specific implementation of how to fill data in.
-
- See:
- :class:`DeadlineJobInfo`
-
- Returns:
- :class:`DeadlineJobInfo`: Filled Deadline JobInfo.
-
- """
- pass
-
- @abstractmethod
- def get_plugin_info(self):
- """Return filled Deadline PluginInfo.
-
- This is host/plugin specific implementation of how to fill data in.
-
- See:
- :class:`DeadlineJobInfo`
-
- Returns:
- dict: Filled Deadline JobInfo.
-
- """
- pass
-
- def get_aux_files(self):
- """Return list of auxiliary files for Deadline job.
-
- If needed this should be overriden, otherwise return empty list as
- that field even empty must be present on Deadline submission.
-
- Returns:
- list: List of files.
-
- """
- return []
-
- def from_published_scene(self, replace_in_path=True):
- """Switch work scene for published scene.
-
- If rendering/exporting from published scenes is enabled, this will
- replace paths from working scene to published scene.
-
- Args:
- replace_in_path (bool): if True, it will try to find
- old scene name in path of expected files and replace it
- with name of published scene.
-
- Returns:
- str: Published scene path.
- None: if no published scene is found.
-
- Note:
- Published scene path is actually determined from project Anatomy
- as at the time this plugin is running scene can still no be
- published.
-
- """
- anatomy = self._instance.context.data['anatomy']
- file_path = None
- for i in self._instance.context:
- if "workfile" in i.data["families"] \
- or i.data["family"] == "workfile":
- # test if there is instance of workfile waiting
- # to be published.
- assert i.data["publish"] is True, (
- "Workfile (scene) must be published along")
- # determine published path from Anatomy.
- template_data = i.data.get("anatomyData")
- rep = i.data.get("representations")[0].get("ext")
- template_data["representation"] = rep
- template_data["ext"] = rep
- template_data["comment"] = None
- anatomy_filled = anatomy.format(template_data)
- template_filled = anatomy_filled["publish"]["path"]
- file_path = os.path.normpath(template_filled)
-
- self.log.info("Using published scene for render {}".format(
- file_path))
-
- if not os.path.exists(file_path):
- self.log.error("published scene does not exist!")
- raise
-
- if not replace_in_path:
- return file_path
-
- # now we need to switch scene in expected files
- # because token will now point to published
- # scene file and that might differ from current one
- new_scene = os.path.splitext(
- os.path.basename(file_path))[0]
- orig_scene = os.path.splitext(
- os.path.basename(
- self._instance.context.data["currentFile"]))[0]
- exp = self._instance.data.get("expectedFiles")
-
- if isinstance(exp[0], dict):
- # we have aovs and we need to iterate over them
- new_exp = {}
- for aov, files in exp[0].items():
- replaced_files = []
- for f in files:
- replaced_files.append(
- f.replace(orig_scene, new_scene)
- )
- new_exp[aov] = replaced_files
- # [] might be too much here, TODO
- self._instance.data["expectedFiles"] = [new_exp]
- else:
- new_exp = []
- for f in exp:
- new_exp.append(
- f.replace(orig_scene, new_scene)
- )
- self._instance.data["expectedFiles"] = new_exp
-
- self.log.info("Scene name was switched {} -> {}".format(
- orig_scene, new_scene
- ))
-
- return file_path
-
- def assemble_payload(
- self, job_info=None, plugin_info=None, aux_files=None):
- """Assemble payload data from its various parts.
-
- Args:
- job_info (DeadlineJobInfo): Deadline JobInfo. You can use
- :class:`DeadlineJobInfo` for it.
- plugin_info (dict): Deadline PluginInfo. Plugin specific options.
- aux_files (list, optional): List of auxiliary file to submit with
- the job.
-
- Returns:
- dict: Deadline Payload.
-
- """
- job = job_info or self.job_info
- return {
- "JobInfo": job.serialize(),
- "PluginInfo": plugin_info or self.plugin_info,
- "AuxFiles": aux_files or self.aux_files
- }
-
- def submit(self, payload):
- """Submit payload to Deadline API end-point.
-
- This takes payload in the form of JSON file and POST it to
- Deadline jobs end-point.
-
- Args:
- payload (dict): dict to become json in deadline submission.
-
- Returns:
- str: resulting Deadline job id.
-
- Throws:
- RuntimeError: if submission fails.
-
- """
- url = "{}/api/jobs".format(self._deadline_url)
- response = requests_post(url, json=payload)
- if not response.ok:
- self.log.error("Submission failed!")
- self.log.error(response.status_code)
- self.log.error(response.content)
- self.log.debug(payload)
- raise RuntimeError(response.text)
-
- result = response.json()
- # for submit publish job
- self._instance.data["deadlineSubmissionJob"] = result
-
- return result["_id"]
diff --git a/openpype/lib/anatomy.py b/openpype/lib/anatomy.py
index 66ecbd66d15..6d339f058fd 100644
--- a/openpype/lib/anatomy.py
+++ b/openpype/lib/anatomy.py
@@ -1,1686 +1,38 @@
-import os
-import re
-import copy
-import platform
-import collections
-import numbers
+"""Code related to project Anatomy was moved
+to 'openpype.pipeline.anatomy' please change your imports as soon as
+possible. File will be probably removed in OpenPype 3.14.*
+"""
-from openpype.settings.lib import (
- get_default_anatomy_settings,
- get_anatomy_settings
-)
-from .log import PypeLogger
+import warnings
+import functools
-log = PypeLogger().get_logger(__name__)
-try:
- StringType = basestring
-except NameError:
- StringType = str
+class AnatomyDeprecatedWarning(DeprecationWarning):
+ pass
-def merge_dict(main_dict, enhance_dict):
- """Merges dictionaries by keys.
+def anatomy_deprecated(func):
+ """Mark functions as deprecated.
- Function call itself if value on key is again dictionary.
-
- Args:
- main_dict (dict): First dict to merge second one into.
- enhance_dict (dict): Second dict to be merged.
-
- Returns:
- dict: Merged result.
-
- .. note:: does not overrides whole value on first found key
- but only values differences from enhance_dict
-
- """
- for key, value in enhance_dict.items():
- if key not in main_dict:
- main_dict[key] = value
- elif isinstance(value, dict) and isinstance(main_dict[key], dict):
- main_dict[key] = merge_dict(main_dict[key], value)
- else:
- main_dict[key] = value
- return main_dict
-
-
-class ProjectNotSet(Exception):
- """Exception raised when is created Anatomy without project name."""
-
-
-class RootCombinationError(Exception):
- """This exception is raised when templates has combined root types."""
-
- def __init__(self, roots):
- joined_roots = ", ".join(
- ["\"{}\"".format(_root) for _root in roots]
- )
- # TODO better error message
- msg = (
- "Combination of root with and"
- " without root name in Templates. {}"
- ).format(joined_roots)
-
- super(RootCombinationError, self).__init__(msg)
-
-
-class Anatomy:
- """Anatomy module helps to keep project settings.
-
- Wraps key project specifications, Templates and Roots.
-
- Args:
- project_name (str): Project name to look on overrides.
+ It will result in a warning being emitted when the function is used.
"""
- root_key_regex = re.compile(r"{(root?[^}]+)}")
- root_name_regex = re.compile(r"root\[([^]]+)\]")
-
- def __init__(self, project_name=None, site_name=None):
- if not project_name:
- project_name = os.environ.get("AVALON_PROJECT")
-
- if not project_name:
- raise ProjectNotSet((
- "Implementation bug: Project name is not set. Anatomy requires"
- " to load data for specific project."
- ))
-
- self.project_name = project_name
-
- self._data = self._prepare_anatomy_data(
- get_anatomy_settings(project_name, site_name)
- )
- self._site_name = site_name
- self._templates_obj = Templates(self)
- self._roots_obj = Roots(self)
-
- # Anatomy used as dictionary
- # - implemented only getters returning copy
- def __getitem__(self, key):
- return copy.deepcopy(self._data[key])
-
- def get(self, key, default=None):
- return copy.deepcopy(self._data).get(key, default)
-
- def keys(self):
- return copy.deepcopy(self._data).keys()
-
- def values(self):
- return copy.deepcopy(self._data).values()
-
- def items(self):
- return copy.deepcopy(self._data).items()
-
- @staticmethod
- def default_data():
- """Default project anatomy data.
-
- Always return fresh loaded data. May be used as data for new project.
-
- Not used inside Anatomy itself.
- """
- return get_default_anatomy_settings(clear_metadata=False)
-
- @staticmethod
- def _prepare_anatomy_data(anatomy_data):
- """Prepare anatomy data for futher processing.
-
- Method added to replace `{task}` with `{task[name]}` in templates.
- """
- templates_data = anatomy_data.get("templates")
- if templates_data:
- # Replace `{task}` with `{task[name]}` in templates
- value_queue = collections.deque()
- value_queue.append(templates_data)
- while value_queue:
- item = value_queue.popleft()
- if not isinstance(item, dict):
- continue
-
- for key in tuple(item.keys()):
- value = item[key]
- if isinstance(value, dict):
- value_queue.append(value)
-
- elif isinstance(value, StringType):
- item[key] = value.replace("{task}", "{task[name]}")
- return anatomy_data
-
- def reset(self):
- """Reset values of cached data in templates and roots objects."""
- self._data = self._prepare_anatomy_data(
- get_anatomy_settings(self.project_name, self._site_name)
- )
- self.templates_obj.reset()
- self.roots_obj.reset()
-
- @property
- def templates(self):
- """Wrap property `templates` of Anatomy's Templates instance."""
- return self._templates_obj.templates
-
- @property
- def templates_obj(self):
- """Return `Templates` object of current Anatomy instance."""
- return self._templates_obj
-
- def format(self, *args, **kwargs):
- """Wrap `format` method of Anatomy's `templates_obj`."""
- return self._templates_obj.format(*args, **kwargs)
-
- def format_all(self, *args, **kwargs):
- """Wrap `format_all` method of Anatomy's `templates_obj`."""
- return self._templates_obj.format_all(*args, **kwargs)
-
- @property
- def roots(self):
- """Wrap `roots` property of Anatomy's `roots_obj`."""
- return self._roots_obj.roots
-
- @property
- def roots_obj(self):
- """Return `Roots` object of current Anatomy instance."""
- return self._roots_obj
-
- def root_environments(self):
- """Return OPENPYPE_ROOT_* environments for current project in dict."""
- return self._roots_obj.root_environments()
-
- def root_environmets_fill_data(self, template=None):
- """Environment variable values in dictionary for rootless path.
-
- Args:
- template (str): Template for environment variable key fill.
- By default is set to `"${}"`.
- """
- return self.roots_obj.root_environmets_fill_data(template)
-
- def find_root_template_from_path(self, *args, **kwargs):
- """Wrapper for Roots `find_root_template_from_path`."""
- return self.roots_obj.find_root_template_from_path(*args, **kwargs)
-
- def path_remapper(self, *args, **kwargs):
- """Wrapper for Roots `path_remapper`."""
- return self.roots_obj.path_remapper(*args, **kwargs)
-
- def all_root_paths(self):
- """Wrapper for Roots `all_root_paths`."""
- return self.roots_obj.all_root_paths()
-
- def set_root_environments(self):
- """Set OPENPYPE_ROOT_* environments for current project."""
- self._roots_obj.set_root_environments()
-
- def root_names(self):
- """Return root names for current project."""
- return self.root_names_from_templates(self.templates)
-
- def _root_keys_from_templates(self, data):
- """Extract root key from templates in data.
-
- Args:
- data (dict): Data that may contain templates as string.
-
- Return:
- set: Set of all root names from templates as strings.
-
- Output example: `{"root[work]", "root[publish]"}`
- """
-
- output = set()
- if isinstance(data, dict):
- for value in data.values():
- for root in self._root_keys_from_templates(value):
- output.add(root)
-
- elif isinstance(data, str):
- for group in re.findall(self.root_key_regex, data):
- output.add(group)
-
- return output
-
- def root_value_for_template(self, template):
- """Returns value of root key from template."""
- root_templates = []
- for group in re.findall(self.root_key_regex, template):
- root_templates.append("{" + group + "}")
-
- if not root_templates:
- return None
-
- return root_templates[0].format(**{"root": self.roots})
-
- def root_names_from_templates(self, templates):
- """Extract root names form anatomy templates.
-
- Returns None if values in templates contain only "{root}".
- Empty list is returned if there is no "root" in templates.
- Else returns all root names from templates in list.
-
- RootCombinationError is raised when templates contain both root types,
- basic "{root}" and with root name specification "{root[work]}".
-
- Args:
- templates (dict): Anatomy templates where roots are not filled.
-
- Return:
- list/None: List of all root names from templates as strings when
- multiroot setup is used, otherwise None is returned.
- """
- roots = list(self._root_keys_from_templates(templates))
- # Return empty list if no roots found in templates
- if not roots:
- return roots
-
- # Raise exception when root keys have roots with and without root name.
- # Invalid output example: ["root", "root[project]", "root[render]"]
- if len(roots) > 1 and "root" in roots:
- raise RootCombinationError(roots)
-
- # Return None if "root" without root name in templates
- if len(roots) == 1 and roots[0] == "root":
- return None
-
- names = set()
- for root in roots:
- for group in re.findall(self.root_name_regex, root):
- names.add(group)
- return list(names)
-
- def fill_root(self, template_path):
- """Fill template path where is only "root" key unfilled.
-
- Args:
- template_path (str): Path with "root" key in.
- Example path: "{root}/projects/MyProject/Shot01/Lighting/..."
-
- Return:
- str: formatted path
- """
- # NOTE does not care if there are different keys than "root"
- return template_path.format(**{"root": self.roots})
-
- @classmethod
- def fill_root_with_path(cls, rootless_path, root_path):
- """Fill path without filled "root" key with passed path.
-
- This is helper to fill root with different directory path than anatomy
- has defined no matter if is single or multiroot.
-
- Output path is same as input path if `rootless_path` does not contain
- unfilled root key.
-
- Args:
- rootless_path (str): Path without filled "root" key. Example:
- "{root[work]}/MyProject/..."
- root_path (str): What should replace root key in `rootless_path`.
-
- Returns:
- str: Path with filled root.
- """
- output = str(rootless_path)
- for group in re.findall(cls.root_key_regex, rootless_path):
- replacement = "{" + group + "}"
- output = output.replace(replacement, root_path)
-
- return output
-
- def replace_root_with_env_key(self, filepath, template=None):
- """Replace root of path with environment key.
-
- # Example:
- ## Project with roots:
- ```
- {
- "nas": {
- "windows": P:/projects",
- ...
- }
- ...
- }
- ```
-
- ## Entered filepath
- "P:/projects/project/asset/task/animation_v001.ma"
-
- ## Entered template
- "<{}>"
-
- ## Output
- "/project/asset/task/animation_v001.ma"
-
- Args:
- filepath (str): Full file path where root should be replaced.
- template (str): Optional template for environment key. Must
- have one index format key.
- Default value if not entered: "${}"
-
- Returns:
- str: Path where root is replaced with environment root key.
-
- Raise:
- ValueError: When project's roots were not found in entered path.
- """
- success, rootless_path = self.find_root_template_from_path(filepath)
- if not success:
- raise ValueError(
- "{}: Project's roots were not found in path: {}".format(
- self.project_name, filepath
- )
- )
-
- data = self.root_environmets_fill_data(template)
- return rootless_path.format(**data)
-
-
-class TemplateMissingKey(Exception):
- """Exception for cases when key does not exist in Anatomy."""
-
- msg = "Anatomy key does not exist: `anatomy{0}`."
-
- def __init__(self, parents):
- parent_join = "".join(["[\"{0}\"]".format(key) for key in parents])
- super(TemplateMissingKey, self).__init__(
- self.msg.format(parent_join)
- )
-
-
-class TemplateUnsolved(Exception):
- """Exception for unsolved template when strict is set to True."""
-
- msg = "Anatomy template \"{0}\" is unsolved.{1}{2}"
- invalid_types_msg = " Keys with invalid DataType: `{0}`."
- missing_keys_msg = " Missing keys: \"{0}\"."
-
- def __init__(self, template, missing_keys, invalid_types):
- invalid_type_items = []
- for _key, _type in invalid_types.items():
- invalid_type_items.append(
- "\"{0}\" {1}".format(_key, str(_type))
- )
-
- invalid_types_msg = ""
- if invalid_type_items:
- invalid_types_msg = self.invalid_types_msg.format(
- ", ".join(invalid_type_items)
- )
-
- missing_keys_msg = ""
- if missing_keys:
- missing_keys_msg = self.missing_keys_msg.format(
- ", ".join(missing_keys)
- )
- super(TemplateUnsolved, self).__init__(
- self.msg.format(template, missing_keys_msg, invalid_types_msg)
- )
-
-
-class TemplateResult(str):
- """Result (formatted template) of anatomy with most of information in.
-
- Args:
- used_values (dict): Dictionary of template filling data with
- only used keys.
- solved (bool): For check if all required keys were filled.
- template (str): Original template.
- missing_keys (list): Missing keys that were not in the data. Include
- missing optional keys.
- invalid_types (dict): When key was found in data, but value had not
- allowed DataType. Allowed data types are `numbers`,
- `str`(`basestring`) and `dict`. Dictionary may cause invalid type
- when value of key in data is dictionary but template expect string
- of number.
- """
-
- def __new__(
- cls, filled_template, template, solved, rootless_path,
- used_values, missing_keys, invalid_types
- ):
- new_obj = super(TemplateResult, cls).__new__(cls, filled_template)
- new_obj.used_values = used_values
- new_obj.solved = solved
- new_obj.template = template
- new_obj.rootless = rootless_path
- new_obj.missing_keys = list(set(missing_keys))
- _invalid_types = {}
- for invalid_type in invalid_types:
- for key, val in invalid_type.items():
- if key in _invalid_types:
- continue
- _invalid_types[key] = val
- new_obj.invalid_types = _invalid_types
- return new_obj
-
-
-class TemplatesDict(dict):
- """Holds and wrap TemplateResults for easy bug report."""
-
- def __init__(self, in_data, key=None, parent=None, strict=None):
- super(TemplatesDict, self).__init__()
- for _key, _value in in_data.items():
- if isinstance(_value, dict):
- _value = self.__class__(_value, _key, self)
- self[_key] = _value
-
- self.key = key
- self.parent = parent
- self.strict = strict
- if self.parent is None and strict is None:
- self.strict = True
-
- def __getitem__(self, key):
- # Raise error about missing key in anatomy.yaml
- if key not in self.keys():
- hier = self.hierarchy()
- hier.append(key)
- raise TemplateMissingKey(hier)
-
- value = super(TemplatesDict, self).__getitem__(key)
- if isinstance(value, self.__class__):
- return value
-
- # Raise exception when expected solved templates and it is not.
- if (
- self.raise_on_unsolved
- and (hasattr(value, "solved") and not value.solved)
- ):
- raise TemplateUnsolved(
- value.template, value.missing_keys, value.invalid_types
- )
- return value
-
- @property
- def raise_on_unsolved(self):
- """To affect this change `strict` attribute."""
- if self.strict is not None:
- return self.strict
- return self.parent.raise_on_unsolved
-
- def hierarchy(self):
- """Return dictionary keys one by one to root parent."""
- if self.parent is None:
- return []
-
- hier_keys = []
- par_hier = self.parent.hierarchy()
- if par_hier:
- hier_keys.extend(par_hier)
- hier_keys.append(self.key)
-
- return hier_keys
-
- @property
- def missing_keys(self):
- """Return missing keys of all children templates."""
- missing_keys = []
- for value in self.values():
- missing_keys.extend(value.missing_keys)
- return list(set(missing_keys))
-
- @property
- def invalid_types(self):
- """Return invalid types of all children templates."""
- invalid_types = {}
- for value in self.values():
- for invalid_type in value.invalid_types:
- _invalid_types = {}
- for key, val in invalid_type.items():
- if key in invalid_types:
- continue
- _invalid_types[key] = val
- invalid_types = merge_dict(invalid_types, _invalid_types)
- return invalid_types
-
- @property
- def used_values(self):
- """Return used values for all children templates."""
- used_values = {}
- for value in self.values():
- used_values = merge_dict(used_values, value.used_values)
- return used_values
-
- def get_solved(self):
- """Get only solved key from templates."""
- result = {}
- for key, value in self.items():
- if isinstance(value, self.__class__):
- value = value.get_solved()
- if not value:
- continue
- result[key] = value
-
- elif (
- not hasattr(value, "solved") or
- value.solved
- ):
- result[key] = value
- return self.__class__(result, key=self.key, parent=self.parent)
-
-
-class Templates:
- key_pattern = re.compile(r"(\{.*?[^{0]*\})")
- key_padding_pattern = re.compile(r"([^:]+)\S+[><]\S+")
- sub_dict_pattern = re.compile(r"([^\[\]]+)")
- optional_pattern = re.compile(r"(<.*?[^{0]*>)[^0-9]*?")
-
- inner_key_pattern = re.compile(r"(\{@.*?[^{}0]*\})")
- inner_key_name_pattern = re.compile(r"\{@(.*?[^{}0]*)\}")
-
- def __init__(self, anatomy):
- self.anatomy = anatomy
- self.loaded_project = None
- self._templates = None
-
- def __getitem__(self, key):
- return self.templates[key]
-
- def get(self, key, default=None):
- return self.templates.get(key, default)
-
- def reset(self):
- self._templates = None
-
- @property
- def project_name(self):
- return self.anatomy.project_name
-
- @property
- def roots(self):
- return self.anatomy.roots
-
- @property
- def templates(self):
- if self.project_name != self.loaded_project:
- self._templates = None
-
- if self._templates is None:
- self._templates = self._discover()
- self.loaded_project = self.project_name
- return self._templates
-
- def default_templates(self):
- """Return default templates data with solved inner keys."""
- return Templates.solve_template_inner_links(
- self.anatomy["templates"]
- )
-
- def _discover(self):
- """ Loads anatomy templates from yaml.
- Default templates are loaded if project is not set or project does
- not have set it's own.
- TODO: create templates if not exist.
-
- Returns:
- TemplatesDict: Contain templates data for current project of
- default templates.
- """
-
- if self.project_name is None:
- # QUESTION create project specific if not found?
- raise AssertionError((
- "Project \"{0}\" does not have his own templates."
- " Trying to use default."
- ).format(self.project_name))
-
- return Templates.solve_template_inner_links(self.anatomy["templates"])
-
- @classmethod
- def replace_inner_keys(cls, matches, value, key_values, key):
- """Replacement of inner keys in template values."""
- for match in matches:
- anatomy_sub_keys = (
- cls.inner_key_name_pattern.findall(match)
- )
- if key in anatomy_sub_keys:
- raise ValueError((
- "Unsolvable recursion in inner keys, "
- "key: \"{}\" is in his own value."
- " Can't determine source, please check Anatomy templates."
- ).format(key))
-
- for anatomy_sub_key in anatomy_sub_keys:
- replace_value = key_values.get(anatomy_sub_key)
- if replace_value is None:
- raise KeyError((
- "Anatomy templates can't be filled."
- " Anatomy key `{0}` has"
- " invalid inner key `{1}`."
- ).format(key, anatomy_sub_key))
-
- valid = isinstance(replace_value, (numbers.Number, StringType))
- if not valid:
- raise ValueError((
- "Anatomy templates can't be filled."
- " Anatomy key `{0}` has"
- " invalid inner key `{1}`"
- " with value `{2}`."
- ).format(key, anatomy_sub_key, str(replace_value)))
-
- value = value.replace(match, str(replace_value))
-
- return value
-
- @classmethod
- def prepare_inner_keys(cls, key_values):
- """Check values of inner keys.
-
- Check if inner key exist in template group and has valid value.
- It is also required to avoid infinite loop with unsolvable recursion
- when first inner key's value refers to second inner key's value where
- first is used.
- """
- keys_to_solve = set(key_values.keys())
- while True:
- found = False
- for key in tuple(keys_to_solve):
- value = key_values[key]
-
- if isinstance(value, StringType):
- matches = cls.inner_key_pattern.findall(value)
- if not matches:
- keys_to_solve.remove(key)
- continue
-
- found = True
- key_values[key] = cls.replace_inner_keys(
- matches, value, key_values, key
- )
- continue
-
- elif not isinstance(value, dict):
- keys_to_solve.remove(key)
- continue
-
- subdict_found = False
- for _key, _value in tuple(value.items()):
- matches = cls.inner_key_pattern.findall(_value)
- if not matches:
- continue
-
- subdict_found = True
- found = True
- key_values[key][_key] = cls.replace_inner_keys(
- matches, _value, key_values,
- "{}.{}".format(key, _key)
- )
-
- if not subdict_found:
- keys_to_solve.remove(key)
-
- if not found:
- break
-
- return key_values
-
- @classmethod
- def solve_template_inner_links(cls, templates):
- """Solve templates inner keys identified by "{@*}".
-
- Process is split into 2 parts.
- First is collecting all global keys (keys in top hierarchy where value
- is not dictionary). All global keys are set for all group keys (keys
- in top hierarchy where value is dictionary). Value of a key is not
- overriden in group if already contain value for the key.
-
- In second part all keys with "at" symbol in value are replaced with
- value of the key afterward "at" symbol from the group.
-
- Args:
- templates (dict): Raw templates data.
-
- Example:
- templates::
- key_1: "value_1",
- key_2: "{@key_1}/{filling_key}"
-
- group_1:
- key_3: "value_3/{@key_2}"
-
- group_2:
- key_2": "value_2"
- key_4": "value_4/{@key_2}"
-
- output::
- key_1: "value_1"
- key_2: "value_1/{filling_key}"
-
- group_1: {
- key_1: "value_1"
- key_2: "value_1/{filling_key}"
- key_3: "value_3/value_1/{filling_key}"
-
- group_2: {
- key_1: "value_1"
- key_2: "value_2"
- key_4: "value_3/value_2"
- """
- default_key_values = templates.pop("defaults", {})
- for key, value in tuple(templates.items()):
- if isinstance(value, dict):
- continue
- default_key_values[key] = templates.pop(key)
-
- # Pop "others" key before before expected keys are processed
- other_templates = templates.pop("others") or {}
-
- keys_by_subkey = {}
- for sub_key, sub_value in templates.items():
- key_values = {}
- key_values.update(default_key_values)
- key_values.update(sub_value)
- keys_by_subkey[sub_key] = cls.prepare_inner_keys(key_values)
-
- for sub_key, sub_value in other_templates.items():
- if sub_key in keys_by_subkey:
- log.warning((
- "Key \"{}\" is duplicated in others. Skipping."
- ).format(sub_key))
- continue
-
- key_values = {}
- key_values.update(default_key_values)
- key_values.update(sub_value)
- keys_by_subkey[sub_key] = cls.prepare_inner_keys(key_values)
-
- default_keys_by_subkeys = cls.prepare_inner_keys(default_key_values)
-
- for key, value in default_keys_by_subkeys.items():
- keys_by_subkey[key] = value
-
- return keys_by_subkey
-
- def _filter_optional(self, template, data):
- """Filter invalid optional keys.
-
- Invalid keys may be missing keys of with invalid value DataType.
-
- Args:
- template (str): Anatomy template which will be formatted.
- data (dict): Containing keys to be filled into template.
-
- Result:
- tuple: Contain origin template without missing optional keys and
- withoud optional keys identificator ("<" and ">"), information
- about missing optional keys and invalid types of optional keys.
-
- """
-
- # Remove optional missing keys
- missing_keys = []
- invalid_types = []
- for optional_group in self.optional_pattern.findall(template):
- _missing_keys = []
- _invalid_types = []
- for optional_key in self.key_pattern.findall(optional_group):
- key = str(optional_key[1:-1])
- key_padding = list(
- self.key_padding_pattern.findall(key)
- )
- if key_padding:
- key = key_padding[0]
-
- validation_result = self._validate_data_key(
- key, data
- )
- missing_key = validation_result["missing_key"]
- invalid_type = validation_result["invalid_type"]
-
- valid = True
- if missing_key is not None:
- _missing_keys.append(missing_key)
- valid = False
-
- if invalid_type is not None:
- _invalid_types.append(invalid_type)
- valid = False
-
- if valid:
- try:
- optional_key.format(**data)
- except KeyError:
- _missing_keys.append(key)
- valid = False
-
- valid = len(_invalid_types) == 0 and len(_missing_keys) == 0
- missing_keys.extend(_missing_keys)
- invalid_types.extend(_invalid_types)
- replacement = ""
- if valid:
- replacement = optional_group[1:-1]
-
- template = template.replace(optional_group, replacement)
- return (template, missing_keys, invalid_types)
-
- def _validate_data_key(self, key, data):
- """Check and prepare missing keys and invalid types of template."""
- result = {
- "missing_key": None,
- "invalid_type": None
- }
-
- # check if key expects subdictionary keys (e.g. project[name])
- key_subdict = list(self.sub_dict_pattern.findall(key))
- used_keys = []
- if len(key_subdict) <= 1:
- if key not in data:
- result["missing_key"] = key
- return result
-
- used_keys.append(key)
- value = data[key]
-
- else:
- value = data
- missing_key = False
- invalid_type = False
- for sub_key in key_subdict:
- if (
- value is None
- or (hasattr(value, "items") and sub_key not in value)
- ):
- missing_key = True
- used_keys.append(sub_key)
- break
-
- elif not hasattr(value, "items"):
- invalid_type = True
- break
-
- used_keys.append(sub_key)
- value = value.get(sub_key)
-
- if missing_key or invalid_type:
- if len(used_keys) == 0:
- invalid_key = key_subdict[0]
- else:
- invalid_key = used_keys[0]
- for idx, sub_key in enumerate(used_keys):
- if idx == 0:
- continue
- invalid_key += "[{0}]".format(sub_key)
-
- if missing_key:
- result["missing_key"] = invalid_key
-
- elif invalid_type:
- result["invalid_type"] = {invalid_key: type(value)}
-
- return result
-
- if isinstance(value, (numbers.Number, Roots, RootItem)):
- return result
-
- for inh_class in type(value).mro():
- if inh_class == StringType:
- return result
-
- result["missing_key"] = key
- result["invalid_type"] = {key: type(value)}
- return result
-
- def _merge_used_values(self, current_used, keys, value):
- key = keys[0]
- _keys = keys[1:]
- if len(_keys) == 0:
- current_used[key] = value
- else:
- next_dict = {}
- if key in current_used:
- next_dict = current_used[key]
- current_used[key] = self._merge_used_values(
- next_dict, _keys, value
- )
- return current_used
-
- def _dict_to_subkeys_list(self, subdict, pre_keys=None):
- if pre_keys is None:
- pre_keys = []
- output = []
- for key in subdict:
- value = subdict[key]
- result = list(pre_keys)
- result.append(key)
- if isinstance(value, dict):
- for item in self._dict_to_subkeys_list(value, result):
- output.append(item)
- else:
- output.append(result)
- return output
-
- def _keys_to_dicts(self, key_list, value):
- if not key_list:
- return None
- if len(key_list) == 1:
- return {key_list[0]: value}
- return {key_list[0]: self._keys_to_dicts(key_list[1:], value)}
-
- def _rootless_path(
- self, template, used_values, final_data, missing_keys, invalid_types
- ):
- if (
- "root" not in used_values
- or "root" in missing_keys
- or "{root" not in template
- ):
- return
-
- for invalid_type in invalid_types:
- if "root" in invalid_type:
- return
-
- root_keys = self._dict_to_subkeys_list({"root": used_values["root"]})
- if not root_keys:
- return
-
- roots_dict = {}
- for used_root_keys in root_keys:
- if not used_root_keys:
- continue
-
- root_key = None
- for key in used_root_keys:
- if root_key is None:
- root_key = key
- else:
- root_key += "[{}]".format(key)
-
- root_key = "{" + root_key + "}"
-
- roots_dict = merge_dict(
- roots_dict,
- self._keys_to_dicts(used_root_keys, root_key)
- )
-
- final_data["root"] = roots_dict["root"]
- return template.format(**final_data)
-
- def _format(self, orig_template, data):
- """ Figure out with whole formatting.
-
- Separate advanced keys (*Like '{project[name]}') from string which must
- be formatted separatelly in case of missing or incomplete keys in data.
-
- Args:
- template (str): Anatomy template which will be formatted.
- data (dict): Containing keys to be filled into template.
-
- Returns:
- TemplateResult: Filled or partially filled template containing all
- data needed or missing for filling template.
- """
- task_data = data.get("task")
- if (
- isinstance(task_data, StringType)
- and "{task[name]}" in orig_template
- ):
- # Change task to dictionary if template expect dictionary
- data["task"] = {"name": task_data}
-
- template, missing_optional, invalid_optional = (
- self._filter_optional(orig_template, data)
- )
- # Remove optional missing keys
- used_values = {}
- invalid_required = []
- missing_required = []
- replace_keys = []
-
- for group in self.key_pattern.findall(template):
- orig_key = group[1:-1]
- key = str(orig_key)
- key_padding = list(self.key_padding_pattern.findall(key))
- if key_padding:
- key = key_padding[0]
-
- validation_result = self._validate_data_key(key, data)
- missing_key = validation_result["missing_key"]
- invalid_type = validation_result["invalid_type"]
-
- if invalid_type is not None:
- invalid_required.append(invalid_type)
- replace_keys.append(key)
- continue
-
- if missing_key is not None:
- missing_required.append(missing_key)
- replace_keys.append(key)
- continue
-
- try:
- value = group.format(**data)
- key_subdict = list(self.sub_dict_pattern.findall(key))
- if len(key_subdict) <= 1:
- used_values[key] = value
-
- else:
- used_values = self._merge_used_values(
- used_values, key_subdict, value
- )
-
- except (TypeError, KeyError):
- missing_required.append(key)
- replace_keys.append(key)
-
- final_data = copy.deepcopy(data)
- for key in replace_keys:
- key_subdict = list(self.sub_dict_pattern.findall(key))
- if len(key_subdict) <= 1:
- final_data[key] = "{" + key + "}"
- continue
-
- replace_key_dst = "---".join(key_subdict)
- replace_key_dst_curly = "{" + replace_key_dst + "}"
- replace_key_src_curly = "{" + key + "}"
- template = template.replace(
- replace_key_src_curly, replace_key_dst_curly
- )
- final_data[replace_key_dst] = replace_key_src_curly
-
- solved = len(missing_required) == 0 and len(invalid_required) == 0
-
- missing_keys = missing_required + missing_optional
- invalid_types = invalid_required + invalid_optional
-
- filled_template = template.format(**final_data)
- # WARNING `_rootless_path` change values in `final_data` please keep
- # in midn when changing order
- rootless_path = self._rootless_path(
- template, used_values, final_data, missing_keys, invalid_types
- )
- if rootless_path is None:
- rootless_path = filled_template
-
- result = TemplateResult(
- filled_template, orig_template, solved, rootless_path,
- used_values, missing_keys, invalid_types
- )
- return result
-
- def solve_dict(self, templates, data):
- """ Solves templates with entered data.
-
- Args:
- templates (dict): All Anatomy templates which will be formatted.
- data (dict): Containing keys to be filled into template.
-
- Returns:
- dict: With `TemplateResult` in values containing filled or
- partially filled templates.
- """
- output = collections.defaultdict(dict)
- for key, orig_value in templates.items():
- if isinstance(orig_value, StringType):
- # Replace {task} by '{task[name]}' for backward compatibility
- if '{task}' in orig_value:
- orig_value = orig_value.replace('{task}', '{task[name]}')
-
- output[key] = self._format(orig_value, data)
- continue
-
- # Check if orig_value has items attribute (any dict inheritance)
- if not hasattr(orig_value, "items"):
- # TODO we should handle this case
- output[key] = orig_value
- continue
-
- for s_key, s_value in self.solve_dict(orig_value, data).items():
- output[key][s_key] = s_value
-
- return output
-
- def format_all(self, in_data, only_keys=True):
- """ Solves templates based on entered data.
-
- Args:
- data (dict): Containing keys to be filled into template.
- only_keys (bool, optional): Decides if environ will be used to
- fill templates or only keys in data.
-
- Returns:
- TemplatesDict: Output `TemplateResult` have `strict` attribute
- set to False so accessing unfilled keys in templates won't
- raise any exceptions.
- """
- output = self.format(in_data, only_keys)
- output.strict = False
- return output
-
- def format(self, in_data, only_keys=True):
- """ Solves templates based on entered data.
-
- Args:
- data (dict): Containing keys to be filled into template.
- only_keys (bool, optional): Decides if environ will be used to
- fill templates or only keys in data.
-
- Returns:
- TemplatesDict: Output `TemplateResult` have `strict` attribute
- set to True so accessing unfilled keys in templates will
- raise exceptions with explaned error.
- """
- # Create a copy of inserted data
- data = copy.deepcopy(in_data)
-
- # Add environment variable to data
- if only_keys is False:
- for key, val in os.environ.items():
- data["$" + key] = val
-
- # override root value
- roots = self.roots
- if roots:
- data["root"] = roots
- solved = self.solve_dict(self.templates, data)
-
- return TemplatesDict(solved)
-
-
-class RootItem:
- """Represents one item or roots.
-
- Holds raw data of root item specification. Raw data contain value
- for each platform, but current platform value is used when object
- is used for formatting of template.
-
- Args:
- root_raw_data (dict): Dictionary containing root values by platform
- names. ["windows", "linux" and "darwin"]
- name (str, optional): Root name which is representing. Used with
- multi root setup otherwise None value is expected.
- parent_keys (list, optional): All dictionary parent keys. Values of
- `parent_keys` are used for get full key which RootItem is
- representing. Used for replacing root value in path with
- formattable key. e.g. parent_keys == ["work"] -> {root[work]}
- parent (object, optional): It is expected to be `Roots` object.
- Value of `parent` won't affect code logic much.
- """
-
- def __init__(
- self, root_raw_data, name=None, parent_keys=None, parent=None
- ):
- lowered_platform_keys = {}
- for key, value in root_raw_data.items():
- lowered_platform_keys[key.lower()] = value
- self.raw_data = lowered_platform_keys
- self.cleaned_data = self._clean_roots(lowered_platform_keys)
- self.name = name
- self.parent_keys = parent_keys or []
- self.parent = parent
-
- self.available_platforms = list(lowered_platform_keys.keys())
- self.value = lowered_platform_keys.get(platform.system().lower())
- self.clean_value = self.clean_root(self.value)
-
- def __format__(self, *args, **kwargs):
- return self.value.__format__(*args, **kwargs)
-
- def __str__(self):
- return str(self.value)
-
- def __repr__(self):
- return self.__str__()
-
- def __getitem__(self, key):
- if isinstance(key, numbers.Number):
- return self.value[key]
-
- additional_info = ""
- if self.parent and self.parent.project_name:
- additional_info += " for project \"{}\"".format(
- self.parent.project_name
- )
-
- raise AssertionError(
- "Root key \"{}\" is missing{}.".format(
- key, additional_info
- )
- )
-
- def full_key(self):
- """Full key value for dictionary formatting in template.
-
- Returns:
- str: Return full replacement key for formatting. This helps when
- multiple roots are set. In that case e.g. `"root[work]"` is
- returned.
- """
- if not self.name:
- return "root"
-
- joined_parent_keys = "".join(
- ["[{}]".format(key) for key in self.parent_keys]
- )
- return "root{}".format(joined_parent_keys)
-
- def clean_path(self, path):
- """Just replace backslashes with forward slashes."""
- return str(path).replace("\\", "/")
-
- def clean_root(self, root):
- """Makes sure root value does not end with slash."""
- if root:
- root = self.clean_path(root)
- while root.endswith("/"):
- root = root[:-1]
- return root
-
- def _clean_roots(self, raw_data):
- """Clean all values of raw root item values."""
- cleaned = {}
- for key, value in raw_data.items():
- cleaned[key] = self.clean_root(value)
- return cleaned
-
- def path_remapper(self, path, dst_platform=None, src_platform=None):
- """Remap path for specific platform.
-
- Args:
- path (str): Source path which need to be remapped.
- dst_platform (str, optional): Specify destination platform
- for which remapping should happen.
- src_platform (str, optional): Specify source platform. This is
- recommended to not use and keep unset until you really want
- to use specific platform.
- roots (dict/RootItem/None, optional): It is possible to remap
- path with different roots then instance where method was
- called has.
-
- Returns:
- str/None: When path does not contain known root then
- None is returned else returns remapped path with "{root}"
- or "{root[]}".
- """
- cleaned_path = self.clean_path(path)
- if dst_platform:
- dst_root_clean = self.cleaned_data.get(dst_platform)
- if not dst_root_clean:
- key_part = ""
- full_key = self.full_key()
- if full_key != "root":
- key_part += "\"{}\" ".format(full_key)
-
- log.warning(
- "Root {}miss platform \"{}\" definition.".format(
- key_part, dst_platform
- )
- )
- return None
-
- if cleaned_path.startswith(dst_root_clean):
- return cleaned_path
-
- if src_platform:
- src_root_clean = self.cleaned_data.get(src_platform)
- if src_root_clean is None:
- log.warning(
- "Root \"{}\" miss platform \"{}\" definition.".format(
- self.full_key(), src_platform
- )
- )
- return None
-
- if not cleaned_path.startswith(src_root_clean):
- return None
-
- subpath = cleaned_path[len(src_root_clean):]
- if dst_platform:
- # `dst_root_clean` is used from upper condition
- return dst_root_clean + subpath
- return self.clean_value + subpath
-
- result, template = self.find_root_template_from_path(path)
- if not result:
- return None
-
- def parent_dict(keys, value):
- if not keys:
- return value
-
- key = keys.pop(0)
- return {key: parent_dict(keys, value)}
-
- if dst_platform:
- format_value = parent_dict(list(self.parent_keys), dst_root_clean)
- else:
- format_value = parent_dict(list(self.parent_keys), self.value)
-
- return template.format(**{"root": format_value})
-
- def find_root_template_from_path(self, path):
- """Replaces known root value with formattable key in path.
-
- All platform values are checked for this replacement.
-
- Args:
- path (str): Path where root value should be found.
-
- Returns:
- tuple: Tuple contain 2 values: `success` (bool) and `path` (str).
- When success it True then path should contain replaced root
- value with formattable key.
-
- Example:
- When input path is::
- "C:/windows/path/root/projects/my_project/file.ext"
-
- And raw data of item looks like::
- {
- "windows": "C:/windows/path/root",
- "linux": "/mount/root"
- }
-
- Output will be::
- (True, "{root}/projects/my_project/file.ext")
-
- If any of raw data value wouldn't match path's root output is::
- (False, "C:/windows/path/root/projects/my_project/file.ext")
- """
- result = False
- output = str(path)
-
- root_paths = list(self.cleaned_data.values())
- mod_path = self.clean_path(path)
- for root_path in root_paths:
- # Skip empty paths
- if not root_path:
- continue
-
- if mod_path.startswith(root_path):
- result = True
- replacement = "{" + self.full_key() + "}"
- output = replacement + mod_path[len(root_path):]
- break
-
- return (result, output)
-
-
-class Roots:
- """Object which should be used for formatting "root" key in templates.
-
- Args:
- anatomy Anatomy: Anatomy object created for a specific project.
- """
-
- env_prefix = "OPENPYPE_PROJECT_ROOT"
- roots_filename = "roots.json"
-
- def __init__(self, anatomy):
- self.anatomy = anatomy
- self.loaded_project = None
- self._roots = None
-
- def __format__(self, *args, **kwargs):
- return self.roots.__format__(*args, **kwargs)
-
- def __getitem__(self, key):
- return self.roots[key]
-
- def reset(self):
- """Reset current roots value."""
- self._roots = None
-
- def path_remapper(
- self, path, dst_platform=None, src_platform=None, roots=None
- ):
- """Remap path for specific platform.
-
- Args:
- path (str): Source path which need to be remapped.
- dst_platform (str, optional): Specify destination platform
- for which remapping should happen.
- src_platform (str, optional): Specify source platform. This is
- recommended to not use and keep unset until you really want
- to use specific platform.
- roots (dict/RootItem/None, optional): It is possible to remap
- path with different roots then instance where method was
- called has.
-
- Returns:
- str/None: When path does not contain known root then
- None is returned else returns remapped path with "{root}"
- or "{root[]}".
- """
- if roots is None:
- roots = self.roots
-
- if roots is None:
- raise ValueError("Roots are not set. Can't find path.")
-
- if "{root" in path:
- path = path.format(**{"root": roots})
- # If `dst_platform` is not specified then return else continue.
- if not dst_platform:
- return path
-
- if isinstance(roots, RootItem):
- return roots.path_remapper(path, dst_platform, src_platform)
-
- for _root in roots.values():
- result = self.path_remapper(
- path, dst_platform, src_platform, _root
- )
- if result is not None:
- return result
-
- def find_root_template_from_path(self, path, roots=None):
- """Find root value in entered path and replace it with formatting key.
-
- Args:
- path (str): Source path where root will be searched.
- roots (Roots/dict, optional): It is possible to use different
- roots than instance where method was triggered has.
-
- Returns:
- tuple: Output contains tuple with bool representing success as
- first value and path with or without replaced root with
- formatting key as second value.
-
- Raises:
- ValueError: When roots are not entered and can't be loaded.
- """
- if roots is None:
- log.debug(
- "Looking for matching root in path \"{}\".".format(path)
- )
- roots = self.roots
-
- if roots is None:
- raise ValueError("Roots are not set. Can't find path.")
-
- if isinstance(roots, RootItem):
- return roots.find_root_template_from_path(path)
-
- for root_name, _root in roots.items():
- success, result = self.find_root_template_from_path(path, _root)
- if success:
- log.info("Found match in root \"{}\".".format(root_name))
- return success, result
-
- log.warning("No matching root was found in current setting.")
- return (False, path)
-
- def set_root_environments(self):
- """Set root environments for current project."""
- for key, value in self.root_environments().items():
- os.environ[key] = value
-
- def root_environments(self):
- """Use root keys to create unique keys for environment variables.
-
- Concatenates prefix "OPENPYPE_ROOT" with root keys to create unique
- keys.
-
- Returns:
- dict: Result is `{(str): (str)}` dicitonary where key represents
- unique key concatenated by keys and value is root value of
- current platform root.
-
- Example:
- With raw root values::
- "work": {
- "windows": "P:/projects/work",
- "linux": "/mnt/share/projects/work",
- "darwin": "/darwin/path/work"
- },
- "publish": {
- "windows": "P:/projects/publish",
- "linux": "/mnt/share/projects/publish",
- "darwin": "/darwin/path/publish"
- }
-
- Result on windows platform::
- {
- "OPENPYPE_ROOT_WORK": "P:/projects/work",
- "OPENPYPE_ROOT_PUBLISH": "P:/projects/publish"
- }
-
- Short example when multiroot is not used::
- {
- "OPENPYPE_ROOT": "P:/projects"
- }
- """
- return self._root_environments()
-
- def all_root_paths(self, roots=None):
- """Return all paths for all roots of all platforms."""
- if roots is None:
- roots = self.roots
-
- output = []
- if isinstance(roots, RootItem):
- for value in roots.raw_data.values():
- output.append(value)
- return output
-
- for _roots in roots.values():
- output.extend(self.all_root_paths(_roots))
- return output
-
- def _root_environments(self, keys=None, roots=None):
- if not keys:
- keys = []
- if roots is None:
- roots = self.roots
-
- if isinstance(roots, RootItem):
- key_items = [self.env_prefix]
- for _key in keys:
- key_items.append(_key.upper())
- key = "_".join(key_items)
- return {key: roots.value}
-
- output = {}
- for _key, _value in roots.items():
- _keys = list(keys)
- _keys.append(_key)
- output.update(self._root_environments(_keys, _value))
- return output
-
- def root_environmets_fill_data(self, template=None):
- """Environment variable values in dictionary for rootless path.
-
- Args:
- template (str): Template for environment variable key fill.
- By default is set to `"${}"`.
- """
- if template is None:
- template = "${}"
- return self._root_environmets_fill_data(template)
-
- def _root_environmets_fill_data(self, template, keys=None, roots=None):
- if keys is None and roots is None:
- return {
- "root": self._root_environmets_fill_data(
- template, [], self.roots
- )
- }
-
- if isinstance(roots, RootItem):
- key_items = [Roots.env_prefix]
- for _key in keys:
- key_items.append(_key.upper())
- key = "_".join(key_items)
- return template.format(key)
-
- output = {}
- for key, value in roots.items():
- _keys = list(keys)
- _keys.append(key)
- output[key] = self._root_environmets_fill_data(
- template, _keys, value
- )
- return output
-
- @property
- def project_name(self):
- """Return project name which will be used for loading root values."""
- return self.anatomy.project_name
-
- @property
- def roots(self):
- """Property for filling "root" key in templates.
-
- This property returns roots for current project or default root values.
- Warning:
- Default roots value may cause issues when project use different
- roots settings. That may happend when project use multiroot
- templates but default roots miss their keys.
- """
- if self.project_name != self.loaded_project:
- self._roots = None
-
- if self._roots is None:
- self._roots = self._discover()
- self.loaded_project = self.project_name
- return self._roots
-
- def _discover(self):
- """ Loads current project's roots or default.
-
- Default roots are loaded if project override's does not contain roots.
-
- Returns:
- `RootItem` or `dict` with multiple `RootItem`s when multiroot
- setting is used.
- """
-
- return self._parse_dict(self.anatomy["roots"], parent=self)
-
- @staticmethod
- def _parse_dict(data, key=None, parent_keys=None, parent=None):
- """Parse roots raw data into RootItem or dictionary with RootItems.
-
- Converting raw roots data to `RootItem` helps to handle platform keys.
- This method is recursive to be able handle multiroot setup and
- is static to be able to load default roots without creating new object.
-
- Args:
- data (dict): Should contain raw roots data to be parsed.
- key (str, optional): Current root key. Set by recursion.
- parent_keys (list): Parent dictionary keys. Set by recursion.
- parent (Roots, optional): Parent object set in `RootItem`
- helps to keep RootItem instance updated with `Roots` object.
-
- Returns:
- `RootItem` or `dict` with multiple `RootItem`s when multiroot
- setting is used.
- """
- if not parent_keys:
- parent_keys = []
- is_last = False
- for value in data.values():
- if isinstance(value, StringType):
- is_last = True
- break
-
- if is_last:
- return RootItem(data, key, parent_keys, parent=parent)
-
- output = {}
- for _key, value in data.items():
- _parent_keys = list(parent_keys)
- _parent_keys.append(_key)
- output[_key] = Roots._parse_dict(value, _key, _parent_keys, parent)
- return output
+ @functools.wraps(func)
+ def new_func(*args, **kwargs):
+ warnings.simplefilter("always", AnatomyDeprecatedWarning)
+ warnings.warn(
+ (
+ "Deprecated import of 'Anatomy'."
+ " Class was moved to 'openpype.pipeline.anatomy'."
+ " Please change your imports of Anatomy in codebase."
+ ),
+ category=AnatomyDeprecatedWarning
+ )
+ return func(*args, **kwargs)
+ return new_func
+
+
+@anatomy_deprecated
+def Anatomy(*args, **kwargs):
+ from openpype.pipeline.anatomy import Anatomy
+ return Anatomy(*args, **kwargs)
diff --git a/openpype/lib/applications.py b/openpype/lib/applications.py
index 86cf0229dfc..e249ae4f1c7 100644
--- a/openpype/lib/applications.py
+++ b/openpype/lib/applications.py
@@ -1,52 +1,53 @@
import os
import sys
-import re
import copy
import json
+import tempfile
import platform
import collections
import inspect
import subprocess
-import distutils.spawn
from abc import ABCMeta, abstractmethod
import six
+from openpype.client import (
+ get_project,
+ get_asset_by_name,
+)
from openpype.settings import (
get_system_settings,
get_project_settings,
- get_environments
+ get_local_settings
)
from openpype.settings.constants import (
METADATA_KEYS,
M_DYNAMIC_KEY_LABEL
)
-from . import (
- PypeLogger,
- Anatomy
-)
+from .log import Logger
from .profiles_filtering import filter_profiles
from .local_settings import get_openpype_username
-from .avalon_context import (
- get_workdir_data,
- get_workdir_with_workdir_data,
- get_workfile_template_key
-)
from .python_module_tools import (
modules_from_path,
classes_from_module
)
-
+from .execute import (
+ find_executable,
+ get_linux_launcher_args
+)
_logger = None
PLATFORM_NAMES = {"windows", "linux", "darwin"}
DEFAULT_ENV_SUBGROUP = "standard"
+CUSTOM_LAUNCH_APP_GROUPS = {
+ "djvview"
+}
def parse_environments(env_data, env_group=None, platform_name=None):
- """Parse environment values from settings byt group and platfrom.
+ """Parse environment values from settings byt group and platform.
Data may contain up to 2 hierarchical levels of dictionaries. At the end
of the last level must be string or list. List is joined using platform
@@ -137,7 +138,7 @@ def get_logger():
"""Global lib.applications logger getter."""
global _logger
if _logger is None:
- _logger = PypeLogger.get_logger(__name__)
+ _logger = Logger.get_logger(__name__)
return _logger
@@ -206,6 +207,7 @@ class ApplicationGroup:
data (dict): Group defying data loaded from settings.
manager (ApplicationManager): Manager that created the group.
"""
+
def __init__(self, name, data, manager):
self.name = name
self.manager = manager
@@ -260,7 +262,7 @@ class Application:
data (dict): Data for the version containing information about
executables, variant label or if is enabled.
Only required key is `executables`.
- group (ApplicationGroup): App group object that created the applicaiton
+ group (ApplicationGroup): App group object that created the application
and under which application belongs.
"""
@@ -369,8 +371,9 @@ class ApplicationManager:
will always use these values. Gives ability to create manager
using different settings.
"""
+
def __init__(self, system_settings=None):
- self.log = PypeLogger.get_logger(self.__class__.__name__)
+ self.log = Logger.get_logger(self.__class__.__name__)
self.app_groups = {}
self.applications = {}
@@ -404,11 +407,47 @@ def refresh(self):
clear_metadata=False, exclude_locals=False
)
+ all_app_defs = {}
+ # Prepare known applications
app_defs = settings["applications"]
+ additional_apps = {}
for group_name, variant_defs in app_defs.items():
if group_name in METADATA_KEYS:
continue
+ if group_name == "additional_apps":
+ additional_apps = variant_defs
+ else:
+ all_app_defs[group_name] = variant_defs
+
+ # Prepare additional applications
+ # - First find dynamic keys that can be used as labels of group
+ dynamic_keys = {}
+ for group_name, variant_defs in additional_apps.items():
+ if group_name == M_DYNAMIC_KEY_LABEL:
+ dynamic_keys = variant_defs
+ break
+
+ # Add additional apps to known applications
+ for group_name, variant_defs in additional_apps.items():
+ if group_name in METADATA_KEYS:
+ continue
+
+ # Determine group label
+ label = variant_defs.get("label")
+ if not label:
+ # Look for label set in dynamic labels
+ label = dynamic_keys.get(group_name)
+ if not label:
+ label = group_name
+ variant_defs["label"] = label
+
+ all_app_defs[group_name] = variant_defs
+
+ for group_name, variant_defs in all_app_defs.items():
+ if group_name in METADATA_KEYS:
+ continue
+
group = ApplicationGroup(group_name, variant_defs, self)
self.app_groups[group_name] = group
for app in group:
@@ -430,6 +469,19 @@ def refresh(self):
for tool in group:
self.tools[tool.full_name] = tool
+ def find_latest_available_variant_for_group(self, group_name):
+ group = self.app_groups.get(group_name)
+ if group is None or not group.enabled:
+ return None
+
+ output = None
+ for _, variant in reversed(sorted(group.variants.items())):
+ executable = variant.find_executable()
+ if executable:
+ output = variant
+ break
+ return output
+
def launch(self, app_name, **data):
"""Launch procedure.
@@ -489,13 +541,13 @@ def __init__(self, name, label, data, manager):
variants = data.get("variants") or {}
label_by_key = variants.pop(M_DYNAMIC_KEY_LABEL, {})
variants_by_name = {}
- for variant_name, variant_env in variants.items():
+ for variant_name, variant_data in variants.items():
if variant_name in METADATA_KEYS:
continue
variant_label = label_by_key.get(variant_name) or variant_name
tool = EnvironmentTool(
- variant_name, variant_label, variant_env, self
+ variant_name, variant_label, variant_data, self
)
variants_by_name[variant_name] = tool
self.variants = variants_by_name
@@ -519,15 +571,30 @@ class EnvironmentTool:
Args:
name (str): Name of the tool.
- environment (dict): Variant environments.
+ variant_data (dict): Variant data with environments and
+ host and app variant filters.
group (str): Name of group which wraps tool.
"""
- def __init__(self, name, label, environment, group):
+ def __init__(self, name, label, variant_data, group):
+ # Backwards compatibility 3.9.1 - 3.9.2
+ # - 'variant_data' contained only environments but contain also host
+ # and application variant filters
+ host_names = variant_data.get("host_names", [])
+ app_variants = variant_data.get("app_variants", [])
+
+ if "environment" in variant_data:
+ environment = variant_data["environment"]
+ else:
+ environment = variant_data
+
+ self.host_names = host_names
+ self.app_variants = app_variants
self.name = name
self.variant_label = label
self.label = " ".join((group.label, label))
self.group = group
+
self._environment = environment
self.full_name = "/".join((group.name, name))
@@ -538,6 +605,19 @@ def __repr__(self):
def environment(self):
return copy.deepcopy(self._environment)
+ def is_valid_for_app(self, app):
+ """Is tool valid for application.
+
+ Args:
+ app (Application): Application for which are prepared environments.
+ """
+ if self.app_variants and app.full_name not in self.app_variants:
+ return False
+
+ if self.host_names and app.host_name not in self.host_names:
+ return False
+ return True
+
class ApplicationExecutable:
"""Representation of executable loaded from settings."""
@@ -592,7 +672,11 @@ def macos_executable_prep(executable):
if os.path.exists(plist_filepath):
import plistlib
- parsed_plist = plistlib.readPlist(plist_filepath)
+ if hasattr(plistlib, "load"):
+ with open(plist_filepath, "rb") as stream:
+ parsed_plist = plistlib.load(stream)
+ else:
+ parsed_plist = plistlib.readPlist(plist_filepath)
executable_filename = parsed_plist.get("CFBundleExecutable")
if executable_filename:
@@ -608,7 +692,7 @@ def as_args(self):
def _realpath(self):
"""Check if path is valid executable path."""
# Check for executable in PATH
- result = distutils.spawn.find_executable(self.executable_path)
+ result = find_executable(self.executable_path)
if result is not None:
return result
@@ -664,7 +748,7 @@ def __init__(self, launch_context):
Always should be called
"""
- self.log = PypeLogger().get_logger(self.__class__.__name__)
+ self.log = Logger.get_logger(self.__class__.__name__)
self.launch_context = launch_context
@@ -774,7 +858,7 @@ class PostLaunchHook(LaunchHook):
class ApplicationLaunchContext:
"""Context of launching application.
- Main purpose of context is to prepare launch arguments and keword arguments
+ Main purpose of context is to prepare launch arguments and keyword arguments
for new process. Most important part of keyword arguments preparations
are environment variables.
@@ -806,7 +890,7 @@ def __init__(self, application, executable, env_group=None, **data):
# Logger
logger_name = "{}-{}".format(self.__class__.__name__, self.app_name)
- self.log = PypeLogger.get_logger(logger_name)
+ self.log = Logger.get_logger(logger_name)
self.executable = executable
@@ -879,6 +963,63 @@ def env(self, value):
)
self.kwargs["env"] = value
+ def _collect_addons_launch_hook_paths(self):
+ """Helper to collect application launch hooks from addons.
+
+ Module have to have implemented 'get_launch_hook_paths' method which
+ can expect appliction as argument or nothing.
+
+ Returns:
+ List[str]: Paths to launch hook directories.
+ """
+
+ expected_types = (list, tuple, set)
+
+ output = []
+ for module in self.modules_manager.get_enabled_modules():
+ # Skip module if does not have implemented 'get_launch_hook_paths'
+ func = getattr(module, "get_launch_hook_paths", None)
+ if func is None:
+ continue
+
+ func = module.get_launch_hook_paths
+ if hasattr(inspect, "signature"):
+ sig = inspect.signature(func)
+ expect_args = len(sig.parameters) > 0
+ else:
+ expect_args = len(inspect.getargspec(func)[0]) > 0
+
+ # Pass application argument if method expect it.
+ try:
+ if expect_args:
+ hook_paths = func(self.application)
+ else:
+ hook_paths = func()
+ except Exception:
+ self.log.warning(
+ "Failed to call 'get_launch_hook_paths'",
+ exc_info=True
+ )
+ continue
+
+ if not hook_paths:
+ continue
+
+ # Convert string to list
+ if isinstance(hook_paths, six.string_types):
+ hook_paths = [hook_paths]
+
+ # Skip invalid types
+ if not isinstance(hook_paths, expected_types):
+ self.log.warning((
+ "Result of `get_launch_hook_paths`"
+ " has invalid type {}. Expected {}"
+ ).format(type(hook_paths), expected_types))
+ continue
+
+ output.extend(hook_paths)
+ return output
+
def paths_to_launch_hooks(self):
"""Directory paths where to look for launch hooks."""
# This method has potential to be part of application manager (maybe).
@@ -886,28 +1027,24 @@ def paths_to_launch_hooks(self):
# TODO load additional studio paths from settings
import openpype
- pype_dir = os.path.dirname(os.path.abspath(openpype.__file__))
-
- # --- START: Backwards compatibility ---
- hooks_dir = os.path.join(pype_dir, "hooks")
-
- subfolder_names = ["global", self.host_name]
- for subfolder_name in subfolder_names:
- path = os.path.join(hooks_dir, subfolder_name)
- if (
- os.path.exists(path)
- and os.path.isdir(path)
- and path not in paths
- ):
- paths.append(path)
- # --- END: Backwards compatibility ---
-
- subfolders_list = (
- ["hooks"],
- ("hosts", self.host_name, "hooks")
- )
- for subfolders in subfolders_list:
- path = os.path.join(pype_dir, *subfolders)
+ openpype_dir = os.path.dirname(os.path.abspath(openpype.__file__))
+
+ global_hooks_dir = os.path.join(openpype_dir, "hooks")
+
+ hooks_dirs = [
+ global_hooks_dir
+ ]
+ if self.host_name:
+ # If host requires launch hooks and is module then launch hooks
+ # should be collected using 'collect_launch_hook_paths'
+ # - module have to implement 'get_launch_hook_paths'
+ host_module = self.modules_manager.get_host_module(self.host_name)
+ if not host_module:
+ hooks_dirs.append(os.path.join(
+ openpype_dir, "hosts", self.host_name, "hooks"
+ ))
+
+ for path in hooks_dirs:
if (
os.path.exists(path)
and os.path.isdir(path)
@@ -916,7 +1053,7 @@ def paths_to_launch_hooks(self):
paths.append(path)
# Load modules paths
- paths.extend(self.modules_manager.collect_launch_hook_paths())
+ paths.extend(self._collect_addons_launch_hook_paths())
return paths
@@ -936,8 +1073,8 @@ def discover_launch_hooks(self, force=False):
self.log.debug("Discovery of launch hooks started.")
paths = self.paths_to_launch_hooks()
- self.log.debug("Paths where will look for launch hooks:{}".format(
- "\n- ".join(paths)
+ self.log.debug("Paths searched for launch hooks:\n{}".format(
+ "\n".join("- {}".format(path) for path in paths)
))
all_classes = {
@@ -947,7 +1084,7 @@ def discover_launch_hooks(self, force=False):
for path in paths:
if not os.path.exists(path):
self.log.info(
- "Path to launch hooks does not exists: \"{}\"".format(path)
+ "Path to launch hooks does not exist: \"{}\"".format(path)
)
continue
@@ -968,13 +1105,14 @@ def discover_launch_hooks(self, force=False):
hook = klass(self)
if not hook.is_valid:
self.log.debug(
- "Hook is not valid for curent launch context."
+ "Skipped hook invalid for current launch context: "
+ "{}".format(klass.__name__)
)
continue
if inspect.isabstract(hook):
self.log.debug("Skipped abstract hook: {}".format(
- str(hook)
+ klass.__name__
))
continue
@@ -986,7 +1124,8 @@ def discover_launch_hooks(self, force=False):
except Exception:
self.log.warning(
- "Initialization of hook failed. {}".format(str(klass)),
+ "Initialization of hook failed: "
+ "{}".format(klass.__name__),
exc_info=True
)
@@ -1022,6 +1161,57 @@ def app_group(self):
def manager(self):
return self.application.manager
+ def _run_process(self):
+ # Windows and MacOS have easier process start
+ low_platform = platform.system().lower()
+ if low_platform in ("windows", "darwin"):
+ return subprocess.Popen(self.launch_args, **self.kwargs)
+
+ # Linux uses mid process
+ # - it is possible that the mid process executable is not
+ # available for this version of OpenPype in that case use standard
+ # launch
+ launch_args = get_linux_launcher_args()
+ if launch_args is None:
+ return subprocess.Popen(self.launch_args, **self.kwargs)
+
+ # Prepare data that will be passed to midprocess
+ # - store arguments to a json and pass path to json as last argument
+ # - pass environments to set
+ app_env = self.kwargs.pop("env", {})
+ json_data = {
+ "args": self.launch_args,
+ "env": app_env
+ }
+ if app_env:
+ # Filter environments of subprocess
+ self.kwargs["env"] = {
+ key: value
+ for key, value in os.environ.items()
+ if key in app_env
+ }
+
+ # Create temp file
+ json_temp = tempfile.NamedTemporaryFile(
+ mode="w", prefix="op_app_args", suffix=".json", delete=False
+ )
+ json_temp.close()
+ json_temp_filpath = json_temp.name
+ with open(json_temp_filpath, "w") as stream:
+ json.dump(json_data, stream)
+
+ launch_args.append(json_temp_filpath)
+
+ # Create mid-process which will launch application
+ process = subprocess.Popen(launch_args, **self.kwargs)
+ # Wait until the process finishes
+ # - This is important! The process would stay in "open" state.
+ process.wait()
+ # Remove the temp file
+ os.remove(json_temp_filpath)
+ # Return process which is already terminated
+ return process
+
def launch(self):
"""Collect data for new process and then create it.
@@ -1058,8 +1248,10 @@ def launch(self):
self.app_name, args_len_str, args
)
)
+ self.launch_args = args
+
# Run process
- self.process = subprocess.Popen(args, **self.kwargs)
+ self.process = self._run_process()
# Process post launch hooks
for postlaunch_hook in self.postlaunch_hooks:
@@ -1068,7 +1260,7 @@ def launch(self):
))
# TODO how to handle errors?
- # - store to variable to let them accesible?
+ # - store to variable to let them accessible?
try:
postlaunch_hook.execute()
@@ -1144,11 +1336,20 @@ def __init__(self, data):
if data.get("env") is None:
data["env"] = os.environ.copy()
+ if "system_settings" not in data:
+ data["system_settings"] = get_system_settings()
+
super(EnvironmentPrepData, self).__init__(data)
def get_app_environments_for_context(
- project_name, asset_name, task_name, app_name, env_group=None, env=None
+ project_name,
+ asset_name,
+ task_name,
+ app_name,
+ env_group=None,
+ env=None,
+ modules_manager=None
):
"""Prepare environment variables by context.
Args:
@@ -1159,11 +1360,14 @@ def get_app_environments_for_context(
by ApplicationManager.
env (dict): Initial environment variables. `os.environ` is used when
not passed.
+ modules_manager (ModulesManager): Initialized modules manager.
Returns:
dict: Environments for passed context and application.
"""
- from avalon.api import AvalonMongoDB
+
+ from openpype.modules import ModulesManager
+ from openpype.pipeline import AvalonMongoDB, Anatomy
# Avalon database connection
dbcon = AvalonMongoDB()
@@ -1171,11 +1375,11 @@ def get_app_environments_for_context(
dbcon.install()
# Project document
- project_doc = dbcon.find_one({"type": "project"})
- asset_doc = dbcon.find_one({
- "type": "asset",
- "name": asset_name
- })
+ project_doc = get_project(project_name)
+ asset_doc = get_asset_by_name(project_name, asset_name)
+
+ if modules_manager is None:
+ modules_manager = ModulesManager()
# Prepare app object which can be obtained only from ApplciationManager
app_manager = ApplicationManager()
@@ -1200,8 +1404,8 @@ def get_app_environments_for_context(
"env": env
})
- prepare_host_environments(data, env_group)
- prepare_context_environments(data, env_group)
+ prepare_app_environments(data, env_group, modules_manager)
+ prepare_context_environments(data, env_group, modules_manager)
# Discard avalon connection
dbcon.uninstall()
@@ -1221,7 +1425,47 @@ def _merge_env(env, current_env):
return result
-def prepare_host_environments(data, env_group=None, implementation_envs=True):
+def _add_python_version_paths(app, env, logger, modules_manager):
+ """Add vendor packages specific for a Python version."""
+
+ for module in modules_manager.get_enabled_modules():
+ module.modify_application_launch_arguments(app, env)
+
+ # Skip adding if host name is not set
+ if not app.host_name:
+ return
+
+ # Add Python 2/3 modules
+ openpype_root = os.getenv("OPENPYPE_REPOS_ROOT")
+ python_vendor_dir = os.path.join(
+ openpype_root,
+ "openpype",
+ "vendor",
+ "python"
+ )
+ if app.use_python_2:
+ pythonpath = os.path.join(python_vendor_dir, "python_2")
+ else:
+ pythonpath = os.path.join(python_vendor_dir, "python_3")
+
+ if not os.path.exists(pythonpath):
+ return
+
+ logger.debug("Adding Python version specific paths to PYTHONPATH")
+ python_paths = [pythonpath]
+
+ # Load PYTHONPATH from current launch context
+ python_path = env.get("PYTHONPATH")
+ if python_path:
+ python_paths.append(python_path)
+
+ # Set new PYTHONPATH to launch context environments
+ env["PYTHONPATH"] = os.pathsep.join(python_paths)
+
+
+def prepare_app_environments(
+ data, env_group=None, implementation_envs=True, modules_manager=None
+):
"""Modify launch environments based on launched app and context.
Args:
@@ -1232,6 +1476,32 @@ def prepare_host_environments(data, env_group=None, implementation_envs=True):
app = data["app"]
log = data["log"]
+ source_env = data["env"].copy()
+
+ if modules_manager is None:
+ from openpype.modules import ModulesManager
+
+ modules_manager = ModulesManager()
+
+ _add_python_version_paths(app, source_env, log, modules_manager)
+
+ # Use environments from local settings
+ filtered_local_envs = {}
+ system_settings = data["system_settings"]
+ whitelist_envs = system_settings["general"].get("local_env_white_list")
+ if whitelist_envs:
+ local_settings = get_local_settings()
+ local_envs = local_settings.get("environments") or {}
+ filtered_local_envs = {
+ key: value
+ for key, value in local_envs.items()
+ if key in whitelist_envs
+ }
+
+ # Apply local environment variables for already existing values
+ for key, value in filtered_local_envs.items():
+ if key in source_env:
+ source_env[key] = value
# `added_env_keys` has debug purpose
added_env_keys = {app.group.name, app.name}
@@ -1249,7 +1519,7 @@ def prepare_host_environments(data, env_group=None, implementation_envs=True):
# Make sure each tool group can be added only once
for key in asset_doc["data"].get("tools_env") or []:
tool = app.manager.tools.get(key)
- if not tool:
+ if not tool or not tool.is_valid_for_app(app):
continue
groups_by_name[tool.group.name] = tool.group
tool_by_group_name[tool.group.name][tool.name] = tool
@@ -1276,17 +1546,28 @@ def prepare_host_environments(data, env_group=None, implementation_envs=True):
# Choose right platform
tool_env = parse_environments(_env_values, env_group)
+
+ # Apply local environment variables
+ # - must happen between all values because they may be used during
+ # merge
+ for key, value in filtered_local_envs.items():
+ if key in tool_env:
+ tool_env[key] = value
+
# Merge dictionaries
env_values = _merge_env(tool_env, env_values)
- merged_env = _merge_env(env_values, data["env"])
+ merged_env = _merge_env(env_values, source_env)
+
loaded_env = acre.compute(merged_env, cleanup=False)
final_env = None
# Add host specific environments
if app.host_name and implementation_envs:
- module = __import__("openpype.hosts", fromlist=[app.host_name])
- host_module = getattr(module, app.host_name, None)
+ host_module = modules_manager.get_host_module(app.host_name)
+ if not host_module:
+ module = __import__("openpype.hosts", fromlist=[app.host_name])
+ host_module = getattr(module, app.host_name, None)
add_implementation_envs = None
if host_module:
add_implementation_envs = getattr(
@@ -1299,7 +1580,7 @@ def prepare_host_environments(data, env_group=None, implementation_envs=True):
if final_env is None:
final_env = loaded_env
- keys_to_remove = set(data["env"].keys()) - set(final_env.keys())
+ keys_to_remove = set(source_env.keys()) - set(final_env.keys())
# Update env
data["env"].update(final_env)
@@ -1312,11 +1593,11 @@ def apply_project_environments_value(
):
"""Apply project specific environments on passed environments.
- The enviornments are applied on passed `env` argument value so it is not
+ The environments are applied on passed `env` argument value so it is not
required to apply changes back.
Args:
- project_name (str): Name of project for which environemnts should be
+ project_name (str): Name of project for which environments should be
received.
env (dict): Environment values on which project specific environments
will be applied.
@@ -1345,13 +1626,16 @@ def apply_project_environments_value(
return env
-def prepare_context_environments(data, env_group=None):
- """Modify launch environemnts with context data for launched host.
+def prepare_context_environments(data, env_group=None, modules_manager=None):
+ """Modify launch environments with context data for launched host.
Args:
data (EnvironmentPrepData): Dictionary where result and intermediate
result will be stored.
"""
+
+ from openpype.pipeline.template_data import get_template_data
+
# Context environments
log = data["log"]
@@ -1372,15 +1656,33 @@ def prepare_context_environments(data, env_group=None):
# Load project specific environments
project_name = project_doc["name"]
project_settings = get_project_settings(project_name)
+ system_settings = get_system_settings()
data["project_settings"] = project_settings
+ data["system_settings"] = system_settings
# Apply project specific environments on current env value
apply_project_environments_value(
project_name, data["env"], project_settings, env_group
)
app = data["app"]
- workdir_data = get_workdir_data(
- project_doc, asset_doc, task_name, app.host_name
+ context_env = {
+ "AVALON_PROJECT": project_doc["name"],
+ "AVALON_ASSET": asset_doc["name"],
+ "AVALON_TASK": task_name,
+ "AVALON_APP_NAME": app.full_name
+ }
+
+ log.debug(
+ "Context environments set:\n{}".format(
+ json.dumps(context_env, indent=4)
+ )
+ )
+ data["env"].update(context_env)
+ if not app.is_host:
+ return
+
+ workdir_data = get_template_data(
+ project_doc, asset_doc, task_name, app.host_name, system_settings
)
data["workdir_data"] = workdir_data
@@ -1391,7 +1693,14 @@ def prepare_context_environments(data, env_group=None):
data["task_type"] = task_type
try:
- workdir = get_workdir_with_workdir_data(workdir_data, anatomy)
+ from openpype.pipeline.workfile import get_workdir_with_workdir_data
+
+ workdir = get_workdir_with_workdir_data(
+ workdir_data,
+ anatomy.project_name,
+ anatomy,
+ project_settings=project_settings
+ )
except Exception as exc:
raise ApplicationLaunchFailed(
@@ -1409,25 +1718,13 @@ def prepare_context_environments(data, env_group=None):
"Couldn't create workdir because: {}".format(str(exc))
)
- context_env = {
- "AVALON_PROJECT": project_doc["name"],
- "AVALON_ASSET": asset_doc["name"],
- "AVALON_TASK": task_name,
- "AVALON_APP": app.host_name,
- "AVALON_APP_NAME": app.full_name,
- "AVALON_WORKDIR": workdir
- }
- log.debug(
- "Context environemnts set:\n{}".format(
- json.dumps(context_env, indent=4)
- )
- )
- data["env"].update(context_env)
+ data["env"]["AVALON_APP"] = app.host_name
+ data["env"]["AVALON_WORKDIR"] = workdir
- _prepare_last_workfile(data, workdir)
+ _prepare_last_workfile(data, workdir, modules_manager)
-def _prepare_last_workfile(data, workdir):
+def _prepare_last_workfile(data, workdir, modules_manager):
"""last workfile workflow preparation.
Function check if should care about last workfile workflow and tries
@@ -1442,9 +1739,15 @@ def _prepare_last_workfile(data, workdir):
result will be stored.
workdir (str): Path to folder where workfiles should be stored.
"""
- import avalon.api
+
+ from openpype.modules import ModulesManager
+ from openpype.pipeline import HOST_WORKFILE_EXTENSIONS
+
+ if not modules_manager:
+ modules_manager = ModulesManager()
log = data["log"]
+
_workdir_data = data.get("workdir_data")
if not _workdir_data:
log.info(
@@ -1458,9 +1761,15 @@ def _prepare_last_workfile(data, workdir):
project_name = data["project_name"]
task_name = data["task_name"]
task_type = data["task_type"]
- start_last_workfile = should_start_last_workfile(
- project_name, app.host_name, task_name, task_type
- )
+
+ start_last_workfile = data.get("start_last_workfile")
+ if start_last_workfile is None:
+ start_last_workfile = should_start_last_workfile(
+ project_name, app.host_name, task_name, task_type
+ )
+ else:
+ log.info("Opening of last workfile was disabled by user")
+
data["start_last_workfile"] = start_last_workfile
workfile_startup = should_workfile_tool_start(
@@ -1484,15 +1793,29 @@ def _prepare_last_workfile(data, workdir):
# Last workfile path
last_workfile_path = data.get("last_workfile_path") or ""
if not last_workfile_path:
- extensions = avalon.api.HOST_WORKFILE_EXTENSIONS.get(app.host_name)
+ host_module = modules_manager.get_host_module(app.host_name)
+ if host_module:
+ extensions = host_module.get_workfile_extensions()
+ else:
+ extensions = HOST_WORKFILE_EXTENSIONS.get(app.host_name)
if extensions:
+ from openpype.pipeline.workfile import (
+ get_workfile_template_key,
+ get_last_workfile
+ )
+
anatomy = data["anatomy"]
+ project_settings = data["project_settings"]
+ task_type = workdir_data["task"]["type"]
+ template_key = get_workfile_template_key(
+ task_type,
+ app.host_name,
+ project_name,
+ project_settings=project_settings
+ )
# Find last workfile
- file_template = anatomy.templates["work"]["file"]
- # Replace {task} by '{task[name]}' for backward compatibility
- if '{task}' in file_template:
- file_template = file_template.replace('{task}', '{task[name]}')
+ file_template = str(anatomy.templates[template_key]["file"])
workdir_data.update({
"version": 1,
@@ -1500,7 +1823,7 @@ def _prepare_last_workfile(data, workdir):
"ext": extensions[0]
})
- last_workfile_path = avalon.api.last_workfile(
+ last_workfile_path = get_last_workfile(
workdir, file_template, workdir_data, extensions, True
)
@@ -1522,7 +1845,7 @@ def should_start_last_workfile(
):
"""Define if host should start last version workfile if possible.
- Default output is `False`. Can be overriden with environment variable
+ Default output is `False`. Can be overridden with environment variable
`AVALON_OPEN_LAST_WORKFILE`, valid values without case sensitivity are
`"0", "1", "true", "false", "yes", "no"`.
@@ -1572,7 +1895,7 @@ def should_workfile_tool_start(
):
"""Define if host should start workfile tool at host launch.
- Default output is `False`. Can be overriden with environment variable
+ Default output is `False`. Can be overridden with environment variable
`OPENPYPE_WORKFILE_TOOL_ON_START`, valid values without case sensitivity are
`"0", "1", "true", "false", "yes", "no"`.
@@ -1615,3 +1938,38 @@ def should_workfile_tool_start(
if output is None:
return default_output
return output
+
+
+def get_non_python_host_kwargs(kwargs, allow_console=True):
+ """Explicit setting of kwargs for Popen for AE/PS/Harmony.
+
+ Expected behavior
+ - openpype_console opens window with logs
+ - openpype_gui has stdout/stderr available for capturing
+
+ Args:
+ kwargs (dict) or None
+ allow_console (bool): use False for inner Popen opening app itself or
+ it will open additional console (at least for Harmony)
+ """
+ if kwargs is None:
+ kwargs = {}
+
+ if platform.system().lower() != "windows":
+ return kwargs
+
+ executable_path = os.environ.get("OPENPYPE_EXECUTABLE")
+ executable_filename = ""
+ if executable_path:
+ executable_filename = os.path.basename(executable_path)
+ if "openpype_gui" in executable_filename:
+ kwargs.update({
+ "creationflags": subprocess.CREATE_NO_WINDOW,
+ "stdout": subprocess.DEVNULL,
+ "stderr": subprocess.DEVNULL
+ })
+ elif allow_console:
+ kwargs.update({
+ "creationflags": subprocess.CREATE_NEW_CONSOLE
+ })
+ return kwargs
diff --git a/openpype/lib/attribute_definitions.py b/openpype/lib/attribute_definitions.py
new file mode 100644
index 00000000000..17658eef931
--- /dev/null
+++ b/openpype/lib/attribute_definitions.py
@@ -0,0 +1,634 @@
+import os
+import re
+import collections
+import uuid
+import json
+from abc import ABCMeta, abstractmethod
+
+import six
+import clique
+
+
+class AbstractAttrDefMeta(ABCMeta):
+ """Meta class to validate existence of 'key' attribute.
+
+ Each object of `AbtractAttrDef` mus have defined 'key' attribute.
+ """
+
+ def __call__(self, *args, **kwargs):
+ obj = super(AbstractAttrDefMeta, self).__call__(*args, **kwargs)
+ init_class = getattr(obj, "__init__class__", None)
+ if init_class is not AbtractAttrDef:
+ raise TypeError("{} super was not called in __init__.".format(
+ type(obj)
+ ))
+ return obj
+
+
+@six.add_metaclass(AbstractAttrDefMeta)
+class AbtractAttrDef:
+ """Abstraction of attribute definiton.
+
+ Each attribute definition must have implemented validation and
+ conversion method.
+
+ Attribute definition should have ability to return "default" value. That
+ can be based on passed data into `__init__` so is not abstracted to
+ attribute.
+
+ QUESTION:
+ How to force to set `key` attribute?
+
+ Args:
+ key(str): Under which key will be attribute value stored.
+ label(str): Attribute label.
+ tooltip(str): Attribute tooltip.
+ is_label_horizontal(bool): UI specific argument. Specify if label is
+ next to value input or ahead.
+ """
+
+ is_value_def = True
+
+ def __init__(
+ self, key, default, label=None, tooltip=None, is_label_horizontal=None
+ ):
+ if is_label_horizontal is None:
+ is_label_horizontal = True
+ self.key = key
+ self.label = label
+ self.tooltip = tooltip
+ self.default = default
+ self.is_label_horizontal = is_label_horizontal
+ self._id = uuid.uuid4()
+
+ self.__init__class__ = AbtractAttrDef
+
+ @property
+ def id(self):
+ return self._id
+
+ def __eq__(self, other):
+ if not isinstance(other, self.__class__):
+ return False
+ return self.key == other.key
+
+ @abstractmethod
+ def convert_value(self, value):
+ """Convert value to a valid one.
+
+ Convert passed value to a valid type. Use default if value can't be
+ converted.
+ """
+
+ pass
+
+
+# -----------------------------------------
+# UI attribute definitoins won't hold value
+# -----------------------------------------
+
+class UIDef(AbtractAttrDef):
+ is_value_def = False
+
+ def __init__(self, key=None, default=None, *args, **kwargs):
+ super(UIDef, self).__init__(key, default, *args, **kwargs)
+
+ def convert_value(self, value):
+ return value
+
+
+class UISeparatorDef(UIDef):
+ pass
+
+
+class UILabelDef(UIDef):
+ def __init__(self, label):
+ super(UILabelDef, self).__init__(label=label)
+
+
+# ---------------------------------------
+# Attribute defintioins should hold value
+# ---------------------------------------
+
+class UnknownDef(AbtractAttrDef):
+ """Definition is not known because definition is not available.
+
+ This attribute can be used to keep existing data unchanged but does not
+ have known definition of type.
+ """
+
+ def __init__(self, key, default=None, **kwargs):
+ kwargs["default"] = default
+ super(UnknownDef, self).__init__(key, **kwargs)
+
+ def convert_value(self, value):
+ return value
+
+
+class NumberDef(AbtractAttrDef):
+ """Number definition.
+
+ Number can have defined minimum/maximum value and decimal points. Value
+ is integer if decimals are 0.
+
+ Args:
+ minimum(int, float): Minimum possible value.
+ maximum(int, float): Maximum possible value.
+ decimals(int): Maximum decimal points of value.
+ default(int, float): Default value for conversion.
+ """
+
+ def __init__(
+ self, key, minimum=None, maximum=None, decimals=None, default=None,
+ **kwargs
+ ):
+ minimum = 0 if minimum is None else minimum
+ maximum = 999999 if maximum is None else maximum
+ # Swap min/max when are passed in opposited order
+ if minimum > maximum:
+ maximum, minimum = minimum, maximum
+
+ if default is None:
+ default = 0
+
+ elif not isinstance(default, (int, float)):
+ raise TypeError((
+ "'default' argument must be 'int' or 'float', not '{}'"
+ ).format(type(default)))
+
+ # Fix default value by mim/max values
+ if default < minimum:
+ default = minimum
+
+ elif default > maximum:
+ default = maximum
+
+ super(NumberDef, self).__init__(key, default=default, **kwargs)
+
+ self.minimum = minimum
+ self.maximum = maximum
+ self.decimals = 0 if decimals is None else decimals
+
+ def __eq__(self, other):
+ if not super(NumberDef, self).__eq__(other):
+ return False
+
+ return (
+ self.decimals == other.decimals
+ and self.maximum == other.maximum
+ and self.maximum == other.maximum
+ )
+
+ def convert_value(self, value):
+ if isinstance(value, six.string_types):
+ try:
+ value = float(value)
+ except Exception:
+ pass
+
+ if not isinstance(value, (int, float)):
+ return self.default
+
+ if self.decimals == 0:
+ return int(value)
+ return round(float(value), self.decimals)
+
+
+class TextDef(AbtractAttrDef):
+ """Text definition.
+
+ Text can have multiline option so endline characters are allowed regex
+ validation can be applied placeholder for UI purposes and default value.
+
+ Regex validation is not part of attribute implemntentation.
+
+ Args:
+ multiline(bool): Text has single or multiline support.
+ regex(str, re.Pattern): Regex validation.
+ placeholder(str): UI placeholder for attribute.
+ default(str, None): Default value. Empty string used when not defined.
+ """
+
+ def __init__(
+ self, key, multiline=None, regex=None, placeholder=None, default=None,
+ **kwargs
+ ):
+ if default is None:
+ default = ""
+
+ super(TextDef, self).__init__(key, default=default, **kwargs)
+
+ if multiline is None:
+ multiline = False
+
+ elif not isinstance(default, six.string_types):
+ raise TypeError((
+ "'default' argument must be a {}, not '{}'"
+ ).format(six.string_types, type(default)))
+
+ if isinstance(regex, six.string_types):
+ regex = re.compile(regex)
+
+ self.multiline = multiline
+ self.placeholder = placeholder
+ self.regex = regex
+
+ def __eq__(self, other):
+ if not super(TextDef, self).__eq__(other):
+ return False
+
+ return (
+ self.multiline == other.multiline
+ and self.regex == other.regex
+ )
+
+ def convert_value(self, value):
+ if isinstance(value, six.string_types):
+ return value
+ return self.default
+
+
+class EnumDef(AbtractAttrDef):
+ """Enumeration of single item from items.
+
+ Args:
+ items: Items definition that can be coverted to
+ `collections.OrderedDict`. Dictionary represent {value: label}
+ relation.
+ default: Default value. Must be one key(value) from passed items.
+ """
+
+ def __init__(self, key, items, default=None, **kwargs):
+ if not items:
+ raise ValueError((
+ "Empty 'items' value. {} must have"
+ " defined values on initialization."
+ ).format(self.__class__.__name__))
+
+ items = collections.OrderedDict(items)
+ if default not in items:
+ for _key in items.keys():
+ default = _key
+ break
+
+ super(EnumDef, self).__init__(key, default=default, **kwargs)
+
+ self.items = items
+
+ def __eq__(self, other):
+ if not super(EnumDef, self).__eq__(other):
+ return False
+
+ if set(self.items.keys()) != set(other.items.keys()):
+ return False
+
+ for key, label in self.items.items():
+ if other.items[key] != label:
+ return False
+ return True
+
+ def convert_value(self, value):
+ if value in self.items:
+ return value
+ return self.default
+
+
+class BoolDef(AbtractAttrDef):
+ """Boolean representation.
+
+ Args:
+ default(bool): Default value. Set to `False` if not defined.
+ """
+
+ def __init__(self, key, default=None, **kwargs):
+ if default is None:
+ default = False
+ super(BoolDef, self).__init__(key, default=default, **kwargs)
+
+ def convert_value(self, value):
+ if isinstance(value, bool):
+ return value
+ return self.default
+
+
+class FileDefItem(object):
+ def __init__(
+ self, directory, filenames, frames=None, template=None
+ ):
+ self.directory = directory
+
+ self.filenames = []
+ self.is_sequence = False
+ self.template = None
+ self.frames = []
+ self.is_empty = True
+
+ self.set_filenames(filenames, frames, template)
+
+ def __str__(self):
+ return json.dumps(self.to_dict())
+
+ def __repr__(self):
+ if self.is_empty:
+ filename = "< empty >"
+ elif self.is_sequence:
+ filename = self.template
+ else:
+ filename = self.filenames[0]
+
+ return "<{}: \"{}\">".format(
+ self.__class__.__name__,
+ os.path.join(self.directory, filename)
+ )
+
+ @property
+ def label(self):
+ if self.is_empty:
+ return None
+
+ if not self.is_sequence:
+ return self.filenames[0]
+
+ frame_start = self.frames[0]
+ filename_template = os.path.basename(self.template)
+ if len(self.frames) == 1:
+ return "{} [{}]".format(filename_template, frame_start)
+
+ frame_end = self.frames[-1]
+ expected_len = (frame_end - frame_start) + 1
+ if expected_len == len(self.frames):
+ return "{} [{}-{}]".format(
+ filename_template, frame_start, frame_end
+ )
+
+ ranges = []
+ _frame_start = None
+ _frame_end = None
+ for frame in range(frame_start, frame_end + 1):
+ if frame not in self.frames:
+ add_to_ranges = _frame_start is not None
+ elif _frame_start is None:
+ _frame_start = _frame_end = frame
+ add_to_ranges = frame == frame_end
+ else:
+ _frame_end = frame
+ add_to_ranges = frame == frame_end
+
+ if add_to_ranges:
+ if _frame_start != _frame_end:
+ _range = "{}-{}".format(_frame_start, _frame_end)
+ else:
+ _range = str(_frame_start)
+ ranges.append(_range)
+ _frame_start = _frame_end = None
+ return "{} [{}]".format(
+ filename_template, ",".join(ranges)
+ )
+
+ def split_sequence(self):
+ if not self.is_sequence:
+ raise ValueError("Cannot split single file item")
+
+ paths = [
+ os.path.join(self.directory, filename)
+ for filename in self.filenames
+ ]
+ return self.from_paths(paths, False)
+
+ @property
+ def ext(self):
+ if self.is_empty:
+ return None
+ _, ext = os.path.splitext(self.filenames[0])
+ if ext:
+ return ext
+ return None
+
+ @property
+ def is_dir(self):
+ if self.is_empty:
+ return False
+
+ # QUESTION a better way how to define folder (in init argument?)
+ if self.ext:
+ return False
+ return True
+
+ def set_directory(self, directory):
+ self.directory = directory
+
+ def set_filenames(self, filenames, frames=None, template=None):
+ if frames is None:
+ frames = []
+ is_sequence = False
+ if frames:
+ is_sequence = True
+
+ if is_sequence and not template:
+ raise ValueError("Missing template for sequence")
+
+ self.is_empty = len(filenames) == 0
+ self.filenames = filenames
+ self.template = template
+ self.frames = frames
+ self.is_sequence = is_sequence
+
+ @classmethod
+ def create_empty_item(cls):
+ return cls("", "")
+
+ @classmethod
+ def from_value(cls, value, allow_sequences):
+ """Convert passed value to FileDefItem objects.
+
+ Returns:
+ list: Created FileDefItem objects.
+ """
+
+ # Convert single item to iterable
+ if not isinstance(value, (list, tuple, set)):
+ value = [value]
+
+ output = []
+ str_filepaths = []
+ for item in value:
+ if isinstance(item, dict):
+ item = cls.from_dict(item)
+
+ if isinstance(item, FileDefItem):
+ if not allow_sequences and item.is_sequence:
+ output.extend(item.split_sequence())
+ else:
+ output.append(item)
+
+ elif isinstance(item, six.string_types):
+ str_filepaths.append(item)
+ else:
+ raise TypeError(
+ "Unknown type \"{}\". Can't convert to {}".format(
+ str(type(item)), cls.__name__
+ )
+ )
+
+ if str_filepaths:
+ output.extend(cls.from_paths(str_filepaths, allow_sequences))
+
+ return output
+
+ @classmethod
+ def from_dict(cls, data):
+ return cls(
+ data["directory"],
+ data["filenames"],
+ data.get("frames"),
+ data.get("template")
+ )
+
+ @classmethod
+ def from_paths(cls, paths, allow_sequences):
+ filenames_by_dir = collections.defaultdict(list)
+ for path in paths:
+ normalized = os.path.normpath(path)
+ directory, filename = os.path.split(normalized)
+ filenames_by_dir[directory].append(filename)
+
+ output = []
+ for directory, filenames in filenames_by_dir.items():
+ if allow_sequences:
+ cols, remainders = clique.assemble(filenames)
+ else:
+ cols = []
+ remainders = filenames
+
+ for remainder in remainders:
+ output.append(cls(directory, [remainder]))
+
+ for col in cols:
+ frames = list(col.indexes)
+ paths = [filename for filename in col]
+ template = col.format("{head}{padding}{tail}")
+
+ output.append(cls(
+ directory, paths, frames, template
+ ))
+
+ return output
+
+ def to_dict(self):
+ output = {
+ "is_sequence": self.is_sequence,
+ "directory": self.directory,
+ "filenames": list(self.filenames),
+ }
+ if self.is_sequence:
+ output.update({
+ "template": self.template,
+ "frames": list(sorted(self.frames)),
+ })
+
+ return output
+
+
+class FileDef(AbtractAttrDef):
+ """File definition.
+ It is possible to define filters of allowed file extensions and if supports
+ folders.
+ Args:
+ single_item(bool): Allow only single path item.
+ folders(bool): Allow folder paths.
+ extensions(List[str]): Allow files with extensions. Empty list will
+ allow all extensions and None will disable files completely.
+ extensions_label(str): Custom label shown instead of extensions in UI.
+ default(str, List[str]): Default value.
+ """
+
+ def __init__(
+ self, key, single_item=True, folders=None, extensions=None,
+ allow_sequences=True, extensions_label=None, default=None, **kwargs
+ ):
+ if folders is None and extensions is None:
+ folders = True
+ extensions = []
+
+ if default is None:
+ if single_item:
+ default = FileDefItem.create_empty_item().to_dict()
+ else:
+ default = []
+ else:
+ if single_item:
+ if isinstance(default, dict):
+ FileDefItem.from_dict(default)
+
+ elif isinstance(default, six.string_types):
+ default = FileDefItem.from_paths([default.strip()])[0]
+
+ else:
+ raise TypeError((
+ "'default' argument must be 'str' or 'dict' not '{}'"
+ ).format(type(default)))
+
+ else:
+ if not isinstance(default, (tuple, list, set)):
+ raise TypeError((
+ "'default' argument must be 'list', 'tuple' or 'set'"
+ ", not '{}'"
+ ).format(type(default)))
+
+ # Change horizontal label
+ is_label_horizontal = kwargs.get("is_label_horizontal")
+ if is_label_horizontal is None:
+ kwargs["is_label_horizontal"] = False
+
+ self.single_item = single_item
+ self.folders = folders
+ self.extensions = set(extensions)
+ self.allow_sequences = allow_sequences
+ self.extensions_label = extensions_label
+ super(FileDef, self).__init__(key, default=default, **kwargs)
+
+ def __eq__(self, other):
+ if not super(FileDef, self).__eq__(other):
+ return False
+
+ return (
+ self.single_item == other.single_item
+ and self.folders == other.folders
+ and self.extensions == other.extensions
+ and self.allow_sequences == other.allow_sequences
+ )
+
+ def convert_value(self, value):
+ if isinstance(value, six.string_types) or isinstance(value, dict):
+ value = [value]
+
+ if isinstance(value, (tuple, list, set)):
+ string_paths = []
+ dict_items = []
+ for item in value:
+ if isinstance(item, six.string_types):
+ string_paths.append(item.strip())
+ elif isinstance(item, dict):
+ try:
+ FileDefItem.from_dict(item)
+ dict_items.append(item)
+ except (ValueError, KeyError):
+ pass
+
+ if string_paths:
+ file_items = FileDefItem.from_paths(string_paths)
+ dict_items.extend([
+ file_item.to_dict()
+ for file_item in file_items
+ ])
+
+ if not self.single_item:
+ return dict_items
+
+ if not dict_items:
+ return self.default
+ return dict_items[0]
+
+ if self.single_item:
+ return FileDefItem.create_empty_item().to_dict()
+ return []
diff --git a/openpype/lib/avalon_context.py b/openpype/lib/avalon_context.py
index cb5bca133da..12f4a5198bd 100644
--- a/openpype/lib/avalon_context.py
+++ b/openpype/lib/avalon_context.py
@@ -1,36 +1,90 @@
"""Should be used only inside of hosts."""
import os
-import json
-import re
import copy
import platform
import logging
-import collections
import functools
-import getpass
+import warnings
-from openpype.settings import get_project_settings
-from .anatomy import Anatomy
+import six
+
+from openpype.client import (
+ get_project,
+ get_assets,
+ get_asset_by_name,
+ get_last_version_by_subset_name,
+ get_workfile_info,
+)
+from openpype.client.operations import (
+ CURRENT_ASSET_DOC_SCHEMA,
+ CURRENT_PROJECT_SCHEMA,
+ CURRENT_PROJECT_CONFIG_SCHEMA,
+ PROJECT_NAME_ALLOWED_SYMBOLS,
+ PROJECT_NAME_REGEX,
+)
from .profiles_filtering import filter_profiles
+from .path_templates import StringTemplate
-# avalon module is not imported at the top
-# - may not be in path at the time of pype.lib initialization
-avalon = None
+legacy_io = None
log = logging.getLogger("AvalonContext")
+# Backwards compatibility - should not be used anymore
+# - Will be removed in OP 3.16.*
CURRENT_DOC_SCHEMAS = {
- "project": "openpype:project-3.0",
- "asset": "openpype:asset-3.0",
- "config": "openpype:config-2.0"
+ "project": CURRENT_PROJECT_SCHEMA,
+ "asset": CURRENT_ASSET_DOC_SCHEMA,
+ "config": CURRENT_PROJECT_CONFIG_SCHEMA
}
-PROJECT_NAME_ALLOWED_SYMBOLS = "a-zA-Z0-9_"
-PROJECT_NAME_REGEX = re.compile(
- "^[{}]+$".format(PROJECT_NAME_ALLOWED_SYMBOLS)
-)
+class AvalonContextDeprecatedWarning(DeprecationWarning):
+ pass
+
+
+def deprecated(new_destination):
+ """Mark functions as deprecated.
+
+ It will result in a warning being emitted when the function is used.
+ """
+
+ func = None
+ if callable(new_destination):
+ func = new_destination
+ new_destination = None
+
+ def _decorator(decorated_func):
+ if new_destination is None:
+ warning_message = (
+ " Please check content of deprecated function to figure out"
+ " possible replacement."
+ )
+ else:
+ warning_message = " Please replace your usage with '{}'.".format(
+ new_destination
+ )
+
+ @functools.wraps(decorated_func)
+ def wrapper(*args, **kwargs):
+ warnings.simplefilter("always", AvalonContextDeprecatedWarning)
+ warnings.warn(
+ (
+ "Call to deprecated function '{}'"
+ "\nFunction was moved or removed.{}"
+ ).format(decorated_func.__name__, warning_message),
+ category=AvalonContextDeprecatedWarning,
+ stacklevel=4
+ )
+ return decorated_func(*args, **kwargs)
+ return wrapper
+
+ if func is None:
+ return _decorator
+ return _decorator(func)
+
+
+@deprecated("openpype.client.operations.create_project")
def create_project(
project_name, project_code, library_project=False, dbcon=None
):
@@ -54,76 +108,27 @@ def create_project(
Returns:
dict: Created project document.
- """
-
- from openpype.settings import ProjectSettings, SaveWarningExc
- from avalon.api import AvalonMongoDB
- from avalon.schema import validate
-
- if dbcon is None:
- dbcon = AvalonMongoDB()
-
- if not PROJECT_NAME_REGEX.match(project_name):
- raise ValueError((
- "Project name \"{}\" contain invalid characters"
- ).format(project_name))
- database = dbcon.database
- project_doc = database[project_name].find_one(
- {"type": "project"},
- {"name": 1}
- )
- if project_doc:
- raise ValueError("Project with name \"{}\" already exists".format(
- project_name
- ))
-
- project_doc = {
- "type": "project",
- "name": project_name,
- "data": {
- "code": project_code,
- "library_project": library_project
- },
- "schema": CURRENT_DOC_SCHEMAS["project"]
- }
- # Insert document with basic data
- database[project_name].insert_one(project_doc)
- # Load ProjectSettings for the project and save it to store all attributes
- # and Anatomy
- try:
- project_settings_entity = ProjectSettings(project_name)
- project_settings_entity.save()
- except SaveWarningExc as exc:
- print(str(exc))
- except Exception:
- database[project_name].delete_one({"type": "project"})
- raise
-
- project_doc = database[project_name].find_one({"type": "project"})
+ Deprecated:
+ Function will be removed after release version 3.16.*
+ """
- try:
- # Validate created project document
- validate(project_doc)
- except Exception:
- # Remove project if is not valid
- database[project_name].delete_one({"type": "project"})
- raise
+ from openpype.client.operations import create_project
- return project_doc
+ return create_project(project_name, project_code, library_project)
-def with_avalon(func):
+def with_pipeline_io(func):
@functools.wraps(func)
- def wrap_avalon(*args, **kwargs):
- global avalon
- if avalon is None:
- import avalon
+ def wrapped(*args, **kwargs):
+ global legacy_io
+ if legacy_io is None:
+ from openpype.pipeline import legacy_io
return func(*args, **kwargs)
- return wrap_avalon
+ return wrapped
-@with_avalon
+@deprecated("openpype.pipeline.context_tools.is_representation_from_latest")
def is_latest(representation):
"""Return whether the representation is from latest version
@@ -133,131 +138,61 @@ def is_latest(representation):
Returns:
bool: Whether the representation is of latest version.
+ Deprecated:
+ Function will be removed after release version 3.15.*
"""
- version = avalon.io.find_one({"_id": representation['parent']})
- if version["type"] == "hero_version":
- return True
-
- # Get highest version under the parent
- highest_version = avalon.io.find_one({
- "type": "version",
- "parent": version["parent"]
- }, sort=[("name", -1)], projection={"name": True})
+ from openpype.pipeline.context_tools import is_representation_from_latest
- if version['name'] == highest_version['name']:
- return True
- else:
- return False
+ return is_representation_from_latest(representation)
-@with_avalon
+@deprecated("openpype.pipeline.load.any_outdated_containers")
def any_outdated():
- """Return whether the current scene has any outdated content"""
-
- checked = set()
- host = avalon.api.registered_host()
- for container in host.ls():
- representation = container['representation']
- if representation in checked:
- continue
-
- representation_doc = avalon.io.find_one(
- {
- "_id": avalon.io.ObjectId(representation),
- "type": "representation"
- },
- projection={"parent": True}
- )
- if representation_doc and not is_latest(representation_doc):
- return True
- elif not representation_doc:
- log.debug("Container '{objectName}' has an invalid "
- "representation, it is missing in the "
- "database".format(**container))
+ """Return whether the current scene has any outdated content.
+
+ Deprecated:
+ Function will be removed after release version 3.15.*
+ """
- checked.add(representation)
+ from openpype.pipeline.load import any_outdated_containers
- return False
+ return any_outdated_containers()
-@with_avalon
+@deprecated("openpype.pipeline.context_tools.get_current_project_asset")
def get_asset(asset_name=None):
""" Returning asset document from database by its name.
- Doesn't count with duplicities on asset names!
+ Doesn't count with duplicities on asset names!
- Args:
- asset_name (str)
+ Args:
+ asset_name (str)
- Returns:
- (MongoDB document)
- """
- if not asset_name:
- asset_name = avalon.api.Session["AVALON_ASSET"]
+ Returns:
+ (MongoDB document)
- asset_document = avalon.io.find_one({
- "name": asset_name,
- "type": "asset"
- })
+ Deprecated:
+ Function will be removed after release version 3.15.*
+ """
- if not asset_document:
- raise TypeError("Entity \"{}\" was not found in DB".format(asset_name))
+ from openpype.pipeline.context_tools import get_current_project_asset
- return asset_document
+ return get_current_project_asset(asset_name=asset_name)
-@with_avalon
-def get_hierarchy(asset_name=None):
+@deprecated("openpype.pipeline.template_data.get_general_template_data")
+def get_system_general_anatomy_data(system_settings=None):
"""
- Obtain asset hierarchy path string from mongo db
-
- Args:
- asset_name (str)
-
- Returns:
- (string): asset hierarchy path
-
+ Deprecated:
+ Function will be removed after release version 3.15.*
"""
- if not asset_name:
- asset_name = avalon.io.Session.get(
- "AVALON_ASSET",
- os.environ["AVALON_ASSET"]
- )
-
- asset_entity = avalon.io.find_one({
- "type": 'asset',
- "name": asset_name
- })
-
- not_set = "PARENTS_NOT_SET"
- entity_parents = asset_entity.get("data", {}).get("parents", not_set)
-
- # If entity already have parents then just return joined
- if entity_parents != not_set:
- return "/".join(entity_parents)
-
- # Else query parents through visualParents and store result to entity
- hierarchy_items = []
- entity = asset_entity
- while True:
- parent_id = entity.get("data", {}).get("visualParent")
- if not parent_id:
- break
- entity = avalon.io.find_one({"_id": parent_id})
- hierarchy_items.append(entity["name"])
-
- # Add parents to entity data for next query
- entity_data = asset_entity.get("data", {})
- entity_data["parents"] = hierarchy_items
- avalon.io.update_many(
- {"_id": asset_entity["_id"]},
- {"$set": {"data": entity_data}}
- )
+ from openpype.pipeline.template_data import get_general_template_data
- return "/".join(hierarchy_items)
+ return get_general_template_data(system_settings)
+@deprecated("openpype.client.get_linked_asset_ids")
def get_linked_asset_ids(asset_doc):
"""Return linked asset ids for `asset_doc` from DB
@@ -266,26 +201,20 @@ def get_linked_asset_ids(asset_doc):
Returns:
(list): MongoDB ids of input links.
+
+ Deprecated:
+ Function will be removed after release version 3.16.*
"""
- output = []
- if not asset_doc:
- return output
-
- input_links = asset_doc["data"].get("inputLinks") or []
- if input_links:
- for item in input_links:
- # Backwards compatibility for "_id" key which was replaced with
- # "id"
- if "_id" in item:
- link_id = item["_id"]
- else:
- link_id = item["id"]
- output.append(link_id)
- return output
+ from openpype.client import get_linked_asset_ids
+ from openpype.pipeline import legacy_io
+
+ project_name = legacy_io.active_project()
+
+ return get_linked_asset_ids(project_name, asset_doc=asset_doc)
-@with_avalon
+@deprecated("openpype.client.get_linked_assets")
def get_linked_assets(asset_doc):
"""Return linked assets for `asset_doc` from DB
@@ -294,15 +223,20 @@ def get_linked_assets(asset_doc):
Returns:
(list) Asset documents of input links for passed asset doc.
+
+ Deprecated:
+ Function will be removed after release version 3.15.*
"""
- link_ids = get_linked_asset_ids(asset_doc)
- if not link_ids:
- return []
- return list(avalon.io.find({"_id": {"$in": link_ids}}))
+ from openpype.pipeline import legacy_io
+ from openpype.client import get_linked_assets
+
+ project_name = legacy_io.active_project()
+ return get_linked_assets(project_name, asset_doc=asset_doc)
-@with_avalon
+
+@deprecated("openpype.client.get_last_version_by_subset_name")
def get_latest_version(asset_name, subset_name, dbcon=None, project_name=None):
"""Retrieve latest version from `asset_name`, and `subset_name`.
@@ -313,68 +247,35 @@ def get_latest_version(asset_name, subset_name, dbcon=None, project_name=None):
Args:
asset_name (str): Name of asset.
subset_name (str): Name of subset.
- dbcon (avalon.mongodb.AvalonMongoDB, optional): Avalon Mongo connection
- with Session.
+ dbcon (AvalonMongoDB, optional): Avalon Mongo connection with Session.
project_name (str, optional): Find latest version in specific project.
Returns:
None: If asset, subset or version were not found.
- dict: Last version document for entered .
+ dict: Last version document for entered.
+
+ Deprecated:
+ Function will be removed after release version 3.15.*
"""
- if not dbcon:
- log.debug("Using `avalon.io` for query.")
- dbcon = avalon.io
- # Make sure is installed
- dbcon.install()
-
- if project_name and project_name != dbcon.Session.get("AVALON_PROJECT"):
- # `avalon.io` has only `_database` attribute
- # but `AvalonMongoDB` has `database`
- database = getattr(dbcon, "database", dbcon._database)
- collection = database[project_name]
- else:
- project_name = dbcon.Session.get("AVALON_PROJECT")
- collection = dbcon
-
- log.debug((
- "Getting latest version for Project: \"{}\" Asset: \"{}\""
- " and Subset: \"{}\""
- ).format(project_name, asset_name, subset_name))
-
- # Query asset document id by asset name
- asset_doc = collection.find_one(
- {"type": "asset", "name": asset_name},
- {"_id": True}
- )
- if not asset_doc:
- log.info(
- "Asset \"{}\" was not found in Database.".format(asset_name)
- )
- return None
+ if not project_name:
+ if not dbcon:
+ from openpype.pipeline import legacy_io
- subset_doc = collection.find_one(
- {"type": "subset", "name": subset_name, "parent": asset_doc["_id"]},
- {"_id": True}
- )
- if not subset_doc:
- log.info(
- "Subset \"{}\" was not found in Database.".format(subset_name)
- )
- return None
+ log.debug("Using `legacy_io` for query.")
+ dbcon = legacy_io
+ # Make sure is installed
+ dbcon.install()
+
+ project_name = dbcon.active_project()
- version_doc = collection.find_one(
- {"type": "version", "parent": subset_doc["_id"]},
- sort=[("name", -1)],
+ return get_last_version_by_subset_name(
+ project_name, subset_name, asset_name=asset_name
)
- if not version_doc:
- log.info(
- "Subset \"{}\" does not have any version yet.".format(subset_name)
- )
- return None
- return version_doc
+@deprecated(
+ "openpype.pipeline.workfile.get_workfile_template_key_from_context")
def get_workfile_template_key_from_context(
asset_name, task_name, host_name, project_name=None,
dbcon=None, project_settings=None
@@ -402,39 +303,30 @@ def get_workfile_template_key_from_context(
Raises:
ValueError: When both 'dbcon' and 'project_name' were not
passed.
+
+ Deprecated:
+ Function will be removed after release version 3.16.*
"""
- if not dbcon:
- if not project_name:
+
+ from openpype.pipeline.workfile import (
+ get_workfile_template_key_from_context
+ )
+
+ if not project_name:
+ if not dbcon:
raise ValueError((
"`get_workfile_template_key_from_context` requires to pass"
" one of 'dbcon' or 'project_name' arguments."
))
- from avalon.api import AvalonMongoDB
-
- dbcon = AvalonMongoDB()
- dbcon.Session["AVALON_PROJECT"] = project_name
+ project_name = dbcon.active_project()
- elif not project_name:
- project_name = dbcon.Session["AVALON_PROJECT"]
-
- asset_doc = dbcon.find_one(
- {
- "type": "asset",
- "name": asset_name
- },
- {
- "data.tasks": 1
- }
- )
- asset_tasks = asset_doc.get("data", {}).get("tasks") or {}
- task_info = asset_tasks.get(task_name) or {}
- task_type = task_info.get("type")
-
- return get_workfile_template_key(
- task_type, host_name, project_name, project_settings
+ return get_workfile_template_key_from_context(
+ asset_name, task_name, host_name, project_name, project_settings
)
+@deprecated(
+ "openpype.pipeline.workfile.get_workfile_template_key")
def get_workfile_template_key(
task_type, host_name, project_name=None, project_settings=None
):
@@ -443,7 +335,7 @@ def get_workfile_template_key(
Function is using profiles from project settings to return right template
for passet task type and host name.
- One of 'project_name' or 'project_settings' must be passed it is preffered
+ One of 'project_name' or 'project_settings' must be passed it is preferred
to pass settings if are already available.
Args:
@@ -457,44 +349,19 @@ def get_workfile_template_key(
Raises:
ValueError: When both 'project_name' and 'project_settings' were not
passed.
- """
- default = "work"
- if not task_type or not host_name:
- return default
-
- if not project_settings:
- if not project_name:
- raise ValueError((
- "`get_workfile_template_key` requires to pass"
- " one of 'project_name' or 'project_settings' arguments."
- ))
- project_settings = get_project_settings(project_name)
- try:
- profiles = (
- project_settings
- ["global"]
- ["tools"]
- ["Workfiles"]
- ["workfile_template_profiles"]
- )
- except Exception:
- profiles = []
+ Deprecated:
+ Function will be removed after release version 3.16.*
+ """
- if not profiles:
- return default
+ from openpype.pipeline.workfile import get_workfile_template_key
- profile_filter = {
- "task_types": task_type,
- "hosts": host_name
- }
- profile = filter_profiles(profiles, profile_filter)
- if profile:
- return profile["workfile_template"] or default
- return default
+ return get_workfile_template_key(
+ task_type, host_name, project_name, project_settings
+ )
-# TODO rename function as is not just "work" specific
+@deprecated("openpype.pipeline.template_data.get_template_data")
def get_workdir_data(project_doc, asset_doc, task_name, host_name):
"""Prepare data for workdir template filling from entered information.
@@ -507,45 +374,26 @@ def get_workdir_data(project_doc, asset_doc, task_name, host_name):
Returns:
dict: Data prepared for filling workdir template.
- """
- task_type = asset_doc['data']['tasks'].get(task_name, {}).get('type')
-
- project_task_types = project_doc["config"]["tasks"]
- task_code = project_task_types.get(task_type, {}).get("short_name")
- asset_parents = asset_doc["data"]["parents"]
- hierarchy = "/".join(asset_parents)
+ Deprecated:
+ Function will be removed after release version 3.15.*
+ """
- parent_name = project_doc["name"]
- if asset_parents:
- parent_name = asset_parents[-1]
+ from openpype.pipeline.template_data import get_template_data
- data = {
- "project": {
- "name": project_doc["name"],
- "code": project_doc["data"].get("code")
- },
- "task": {
- "name": task_name,
- "type": task_type,
- "short": task_code,
- },
- "asset": asset_doc["name"],
- "parent": parent_name,
- "app": host_name,
- "user": getpass.getuser(),
- "hierarchy": hierarchy,
- }
- return data
+ return get_template_data(
+ project_doc, asset_doc, task_name, host_name
+ )
+@deprecated("openpype.pipeline.workfile.get_workdir_with_workdir_data")
def get_workdir_with_workdir_data(
workdir_data, anatomy=None, project_name=None, template_key=None
):
"""Fill workdir path from entered data and project's anatomy.
It is possible to pass only project's name instead of project's anatomy but
- one of them **must** be entered. It is preffered to enter anatomy if is
+ one of them **must** be entered. It is preferred to enter anatomy if is
available as initialization of a new Anatomy object may be time consuming.
Args:
@@ -564,28 +412,28 @@ def get_workdir_with_workdir_data(
Raises:
ValueError: When both `anatomy` and `project_name` are set to None.
+
+ Deprecated:
+ Function will be removed after release version 3.15.*
"""
+
if not anatomy and not project_name:
raise ValueError((
"Missing required arguments one of `project_name` or `anatomy`"
" must be entered."
))
- if not anatomy:
- anatomy = Anatomy(project_name)
+ if not project_name:
+ project_name = anatomy.project_name
- if not template_key:
- template_key = get_workfile_template_key(
- workdir_data["task"]["type"],
- workdir_data["app"],
- project_name=workdir_data["project"]["name"]
- )
+ from openpype.pipeline.workfile import get_workdir_with_workdir_data
- anatomy_filled = anatomy.format(workdir_data)
- # Output is TemplateResult object which contain usefull data
- return anatomy_filled[template_key]["folder"]
+ return get_workdir_with_workdir_data(
+ workdir_data, project_name, anatomy, template_key
+ )
+@deprecated("openpype.pipeline.workfile.get_workdir_with_workdir_data")
def get_workdir(
project_doc,
asset_doc,
@@ -604,7 +452,7 @@ def get_workdir(
because workdir template may contain `{app}` key. In `Session`
is stored under `AVALON_APP` key.
anatomy (Anatomy): Optional argument. Anatomy object is created using
- project name from `project_doc`. It is preffered to pass this
+ project name from `project_doc`. It is preferred to pass this
argument as initialization of a new Anatomy object may be time
consuming.
template_key (str): Key of work templates in anatomy templates. Default
@@ -612,20 +460,135 @@ def get_workdir(
Returns:
TemplateResult: Workdir path.
+
+ Deprecated:
+ Function will be removed after release version 3.15.*
"""
- if not anatomy:
- anatomy = Anatomy(project_doc["name"])
- workdir_data = get_workdir_data(
- project_doc, asset_doc, task_name, host_name
+ from openpype.pipeline.workfile import get_workdir
+ # Output is TemplateResult object which contain useful data
+ return get_workdir(
+ project_doc,
+ asset_doc,
+ task_name,
+ host_name,
+ anatomy,
+ template_key
)
- # Output is TemplateResult object which contain usefull data
- return get_workdir_with_workdir_data(
- workdir_data, anatomy, template_key=template_key
+
+
+@deprecated("openpype.pipeline.context_tools.get_template_data_from_session")
+def template_data_from_session(session=None):
+ """ Return dictionary with template from session keys.
+
+ Args:
+ session (dict, Optional): The Session to use. If not provided use the
+ currently active global Session.
+
+ Returns:
+ dict: All available data from session.
+
+ Deprecated:
+ Function will be removed after release version 3.15.*
+ """
+
+ from openpype.pipeline.context_tools import get_template_data_from_session
+
+ return get_template_data_from_session(session)
+
+
+@deprecated("openpype.pipeline.context_tools.compute_session_changes")
+def compute_session_changes(
+ session, task=None, asset=None, app=None, template_key=None
+):
+ """Compute the changes for a Session object on asset, task or app switch
+
+ This does *NOT* update the Session object, but returns the changes
+ required for a valid update of the Session.
+
+ Args:
+ session (dict): The initial session to compute changes to.
+ This is required for computing the full Work Directory, as that
+ also depends on the values that haven't changed.
+ task (str, Optional): Name of task to switch to.
+ asset (str or dict, Optional): Name of asset to switch to.
+ You can also directly provide the Asset dictionary as returned
+ from the database to avoid an additional query. (optimization)
+ app (str, Optional): Name of app to switch to.
+
+ Returns:
+ dict: The required changes in the Session dictionary.
+
+ Deprecated:
+ Function will be removed after release version 3.16.*
+ """
+
+ from openpype.pipeline import legacy_io
+ from openpype.pipeline.context_tools import compute_session_changes
+
+ if isinstance(asset, six.string_types):
+ project_name = legacy_io.active_project()
+ asset = get_asset_by_name(project_name, asset)
+
+ return compute_session_changes(
+ session,
+ asset,
+ task,
+ template_key
)
-@with_avalon
+@deprecated("openpype.pipeline.context_tools.get_workdir_from_session")
+def get_workdir_from_session(session=None, template_key=None):
+ """Calculate workdir path based on session data.
+
+ Args:
+ session (Union[None, Dict[str, str]]): Session to use. If not passed
+ current context session is used (from legacy_io).
+ template_key (Union[str, None]): Precalculate template key to define
+ workfile template name in Anatomy.
+
+ Returns:
+ str: Workdir path.
+
+ Deprecated:
+ Function will be removed after release version 3.16.*
+ """
+
+ from openpype.pipeline.context_tools import get_workdir_from_session
+
+ return get_workdir_from_session(session, template_key)
+
+
+@deprecated("openpype.pipeline.context_tools.change_current_context")
+def update_current_task(task=None, asset=None, app=None, template_key=None):
+ """Update active Session to a new task work area.
+
+ This updates the live Session to a different `asset`, `task` or `app`.
+
+ Args:
+ task (str): The task to set.
+ asset (str): The asset to set.
+ app (str): The app to set.
+
+ Returns:
+ dict: The changed key, values in the current Session.
+
+ Deprecated:
+ Function will be removed after release version 3.16.*
+ """
+
+ from openpype.pipeline import legacy_io
+ from openpype.pipeline.context_tools import change_current_context
+
+ project_name = legacy_io.active_project()
+ if isinstance(asset, six.string_types):
+ asset = get_asset_by_name(project_name, asset)
+
+ return change_current_context(asset, task, template_key)
+
+
+@deprecated("openpype.client.get_workfile_info")
def get_workfile_doc(asset_id, task_name, filename, dbcon=None):
"""Return workfile document for entered context.
@@ -637,24 +600,25 @@ def get_workfile_doc(asset_id, task_name, filename, dbcon=None):
task_name (str): Name of task under which the workfile belongs.
filename (str): Name of a workfile.
dbcon (AvalonMongoDB): Optionally enter avalon AvalonMongoDB object and
- `avalon.io` is used if not entered.
+ `legacy_io` is used if not entered.
Returns:
dict: Workfile document or None.
+
+ Deprecated:
+ Function will be removed after release version 3.15.*
"""
- # Use avalon.io if dbcon is not entered
+
+ # Use legacy_io if dbcon is not entered
if not dbcon:
- dbcon = avalon.io
+ from openpype.pipeline import legacy_io
+ dbcon = legacy_io
- return dbcon.find_one({
- "type": "workfile",
- "parent": asset_id,
- "task_name": task_name,
- "filename": filename
- })
+ project_name = dbcon.active_project()
+ return get_workfile_info(project_name, asset_id, task_name, filename)
-@with_avalon
+@deprecated
def create_workfile_doc(asset_doc, task_name, filename, workdir, dbcon=None):
"""Creates or replace workfile document in mongo.
@@ -667,11 +631,16 @@ def create_workfile_doc(asset_doc, task_name, filename, workdir, dbcon=None):
filename (str): Filename of workfile.
workdir (str): Path to directory where `filename` is located.
dbcon (AvalonMongoDB): Optionally enter avalon AvalonMongoDB object and
- `avalon.io` is used if not entered.
+ `legacy_io` is used if not entered.
"""
- # Use avalon.io if dbcon is not entered
+
+ from openpype.pipeline import Anatomy
+ from openpype.pipeline.template_data import get_template_data
+
+ # Use legacy_io if dbcon is not entered
if not dbcon:
- dbcon = avalon.io
+ from openpype.pipeline import legacy_io
+ dbcon = legacy_io
# Filter of workfile document
doc_filter = {
@@ -684,12 +653,13 @@ def create_workfile_doc(asset_doc, task_name, filename, workdir, dbcon=None):
doc_data = copy.deepcopy(doc_filter)
# Prepare project for workdir data
- project_doc = dbcon.find_one({"type": "project"})
- workdir_data = get_workdir_data(
+ project_name = dbcon.active_project()
+ project_doc = get_project(project_name)
+ workdir_data = get_template_data(
project_doc, asset_doc, task_name, dbcon.Session["AVALON_APP"]
)
# Prepare anatomy
- anatomy = Anatomy(project_doc["name"])
+ anatomy = Anatomy(project_name)
# Get workdir path (result is anatomy.TemplateResult)
template_workdir = get_workdir_with_workdir_data(
workdir_data, anatomy
@@ -716,7 +686,7 @@ def create_workfile_doc(asset_doc, task_name, filename, workdir, dbcon=None):
)
-@with_avalon
+@deprecated
def save_workfile_data_to_doc(workfile_doc, data, dbcon=None):
if not workfile_doc:
# TODO add log message
@@ -725,9 +695,10 @@ def save_workfile_data_to_doc(workfile_doc, data, dbcon=None):
if not data:
return
- # Use avalon.io if dbcon is not entered
+ # Use legacy_io if dbcon is not entered
if not dbcon:
- dbcon = avalon.io
+ from openpype.pipeline import legacy_io
+ dbcon = legacy_io
# Convert data to mongo modification keys/values
# - this is naive implementation which does not expect nested
@@ -744,666 +715,19 @@ def save_workfile_data_to_doc(workfile_doc, data, dbcon=None):
)
-class BuildWorkfile:
- """Wrapper for build workfile process.
+@deprecated("openpype.pipeline.workfile.BuildWorkfile")
+def BuildWorkfile():
+ """Build workfile class was moved to workfile pipeline.
- Load representations for current context by build presets. Build presets
- are host related, since each host has it's loaders.
+ Deprecated:
+ Function will be removed after release version 3.16.*
"""
+ from openpype.pipeline.workfile import BuildWorkfile
- log = logging.getLogger("BuildWorkfile")
-
- @staticmethod
- def map_subsets_by_family(subsets):
- subsets_by_family = collections.defaultdict(list)
- for subset in subsets:
- family = subset["data"].get("family")
- if not family:
- families = subset["data"].get("families")
- if not families:
- continue
- family = families[0]
-
- subsets_by_family[family].append(subset)
- return subsets_by_family
-
- def process(self):
- """Main method of this wrapper.
-
- Building of workfile is triggered and is possible to implement
- post processing of loaded containers if necessary.
- """
- containers = self.build_workfile()
-
- return containers
-
- @with_avalon
- def build_workfile(self):
- """Prepares and load containers into workfile.
-
- Loads latest versions of current and linked assets to workfile by logic
- stored in Workfile profiles from presets. Profiles are set by host,
- filtered by current task name and used by families.
-
- Each family can specify representation names and loaders for
- representations and first available and successful loaded
- representation is returned as container.
-
- At the end you'll get list of loaded containers per each asset.
-
- loaded_containers [{
- "asset_entity": ,
- "containers": [, , ...]
- }, {
- "asset_entity": ,
- "containers": [, ...]
- }, {
- ...
- }]
- """
- # Get current asset name and entity
- current_asset_name = avalon.io.Session["AVALON_ASSET"]
- current_asset_entity = avalon.io.find_one({
- "type": "asset",
- "name": current_asset_name
- })
-
- # Skip if asset was not found
- if not current_asset_entity:
- print("Asset entity with name `{}` was not found".format(
- current_asset_name
- ))
- return
-
- # Prepare available loaders
- loaders_by_name = {}
- for loader in avalon.api.discover(avalon.api.Loader):
- loader_name = loader.__name__
- if loader_name in loaders_by_name:
- raise KeyError(
- "Duplicated loader name {0}!".format(loader_name)
- )
- loaders_by_name[loader_name] = loader
-
- # Skip if there are any loaders
- if not loaders_by_name:
- self.log.warning("There are no registered loaders.")
- return
-
- # Get current task name
- current_task_name = avalon.io.Session["AVALON_TASK"]
-
- # Load workfile presets for task
- self.build_presets = self.get_build_presets(
- current_task_name, current_asset_entity
- )
-
- # Skip if there are any presets for task
- if not self.build_presets:
- self.log.warning(
- "Current task `{}` does not have any loading preset.".format(
- current_task_name
- )
- )
- return
-
- # Get presets for loading current asset
- current_context_profiles = self.build_presets.get("current_context")
- # Get presets for loading linked assets
- link_context_profiles = self.build_presets.get("linked_assets")
- # Skip if both are missing
- if not current_context_profiles and not link_context_profiles:
- self.log.warning(
- "Current task `{}` has empty loading preset.".format(
- current_task_name
- )
- )
- return
-
- elif not current_context_profiles:
- self.log.warning((
- "Current task `{}` doesn't have any loading"
- " preset for it's context."
- ).format(current_task_name))
-
- elif not link_context_profiles:
- self.log.warning((
- "Current task `{}` doesn't have any"
- "loading preset for it's linked assets."
- ).format(current_task_name))
-
- # Prepare assets to process by workfile presets
- assets = []
- current_asset_id = None
- if current_context_profiles:
- # Add current asset entity if preset has current context set
- assets.append(current_asset_entity)
- current_asset_id = current_asset_entity["_id"]
-
- if link_context_profiles:
- # Find and append linked assets if preset has set linked mapping
- link_assets = get_linked_assets(current_asset_entity)
- if link_assets:
- assets.extend(link_assets)
-
- # Skip if there are no assets. This can happen if only linked mapping
- # is set and there are no links for his asset.
- if not assets:
- self.log.warning(
- "Asset does not have linked assets. Nothing to process."
- )
- return
-
- # Prepare entities from database for assets
- prepared_entities = self._collect_last_version_repres(assets)
-
- # Load containers by prepared entities and presets
- loaded_containers = []
- # - Current asset containers
- if current_asset_id and current_asset_id in prepared_entities:
- current_context_data = prepared_entities.pop(current_asset_id)
- loaded_data = self.load_containers_by_asset_data(
- current_context_data, current_context_profiles, loaders_by_name
- )
- if loaded_data:
- loaded_containers.append(loaded_data)
-
- # - Linked assets container
- for linked_asset_data in prepared_entities.values():
- loaded_data = self.load_containers_by_asset_data(
- linked_asset_data, link_context_profiles, loaders_by_name
- )
- if loaded_data:
- loaded_containers.append(loaded_data)
-
- # Return list of loaded containers
- return loaded_containers
-
- @with_avalon
- def get_build_presets(self, task_name, asset_doc):
- """ Returns presets to build workfile for task name.
-
- Presets are loaded for current project set in
- io.Session["AVALON_PROJECT"], filtered by registered host
- and entered task name.
-
- Args:
- task_name (str): Task name used for filtering build presets.
-
- Returns:
- (dict): preset per entered task name
- """
- host_name = avalon.api.registered_host().__name__.rsplit(".", 1)[-1]
- project_settings = get_project_settings(
- avalon.io.Session["AVALON_PROJECT"]
- )
-
- host_settings = project_settings.get(host_name) or {}
- # Get presets for host
- wb_settings = host_settings.get("workfile_builder")
- if not wb_settings:
- # backward compatibility
- wb_settings = host_settings.get("workfile_build") or {}
-
- builder_profiles = wb_settings.get("profiles")
- if not builder_profiles:
- return None
-
- task_type = (
- asset_doc
- .get("data", {})
- .get("tasks", {})
- .get(task_name, {})
- .get("type")
- )
- filter_data = {
- "task_types": task_type,
- "tasks": task_name
- }
- return filter_profiles(builder_profiles, filter_data)
-
- def _filter_build_profiles(self, build_profiles, loaders_by_name):
- """ Filter build profiles by loaders and prepare process data.
-
- Valid profile must have "loaders", "families" and "repre_names" keys
- with valid values.
- - "loaders" expects list of strings representing possible loaders.
- - "families" expects list of strings for filtering
- by main subset family.
- - "repre_names" expects list of strings for filtering by
- representation name.
-
- Lowered "families" and "repre_names" are prepared for each profile with
- all required keys.
-
- Args:
- build_profiles (dict): Profiles for building workfile.
- loaders_by_name (dict): Available loaders per name.
-
- Returns:
- (list): Filtered and prepared profiles.
- """
- valid_profiles = []
- for profile in build_profiles:
- # Check loaders
- profile_loaders = profile.get("loaders")
- if not profile_loaders:
- self.log.warning((
- "Build profile has missing loaders configuration: {0}"
- ).format(json.dumps(profile, indent=4)))
- continue
-
- # Check if any loader is available
- loaders_match = False
- for loader_name in profile_loaders:
- if loader_name in loaders_by_name:
- loaders_match = True
- break
-
- if not loaders_match:
- self.log.warning((
- "All loaders from Build profile are not available: {0}"
- ).format(json.dumps(profile, indent=4)))
- continue
-
- # Check families
- profile_families = profile.get("families")
- if not profile_families:
- self.log.warning((
- "Build profile is missing families configuration: {0}"
- ).format(json.dumps(profile, indent=4)))
- continue
-
- # Check representation names
- profile_repre_names = profile.get("repre_names")
- if not profile_repre_names:
- self.log.warning((
- "Build profile is missing"
- " representation names filtering: {0}"
- ).format(json.dumps(profile, indent=4)))
- continue
-
- # Prepare lowered families and representation names
- profile["families_lowered"] = [
- fam.lower() for fam in profile_families
- ]
- profile["repre_names_lowered"] = [
- name.lower() for name in profile_repre_names
- ]
-
- valid_profiles.append(profile)
-
- return valid_profiles
-
- def _prepare_profile_for_subsets(self, subsets, profiles):
- """Select profile for each subset byt it's data.
-
- Profiles are filtered for each subset individually.
- Profile is filtered by subset's family, optionally by name regex and
- representation names set in profile.
- It is possible to not find matching profile for subset, in that case
- subset is skipped and it is possible that none of subsets have
- matching profile.
-
- Args:
- subsets (list): Subset documents.
- profiles (dict): Build profiles.
-
- Returns:
- (dict) Profile by subset's id.
- """
- # Prepare subsets
- subsets_by_family = self.map_subsets_by_family(subsets)
-
- profiles_per_subset_id = {}
- for family, subsets in subsets_by_family.items():
- family_low = family.lower()
- for profile in profiles:
- # Skip profile if does not contain family
- if family_low not in profile["families_lowered"]:
- continue
-
- # Precompile name filters as regexes
- profile_regexes = profile.get("subset_name_filters")
- if profile_regexes:
- _profile_regexes = []
- for regex in profile_regexes:
- _profile_regexes.append(re.compile(regex))
- profile_regexes = _profile_regexes
-
- # TODO prepare regex compilation
- for subset in subsets:
- # Verify regex filtering (optional)
- if profile_regexes:
- valid = False
- for pattern in profile_regexes:
- if re.match(pattern, subset["name"]):
- valid = True
- break
-
- if not valid:
- continue
-
- profiles_per_subset_id[subset["_id"]] = profile
-
- # break profiles loop on finding the first matching profile
- break
- return profiles_per_subset_id
-
- def load_containers_by_asset_data(
- self, asset_entity_data, build_profiles, loaders_by_name
- ):
- """Load containers for entered asset entity by Build profiles.
-
- Args:
- asset_entity_data (dict): Prepared data with subsets, last version
- and representations for specific asset.
- build_profiles (dict): Build profiles.
- loaders_by_name (dict): Available loaders per name.
-
- Returns:
- (dict) Output contains asset document and loaded containers.
- """
-
- # Make sure all data are not empty
- if not asset_entity_data or not build_profiles or not loaders_by_name:
- return
-
- asset_entity = asset_entity_data["asset_entity"]
-
- valid_profiles = self._filter_build_profiles(
- build_profiles, loaders_by_name
- )
- if not valid_profiles:
- self.log.warning(
- "There are not valid Workfile profiles. Skipping process."
- )
- return
-
- self.log.debug("Valid Workfile profiles: {}".format(valid_profiles))
-
- subsets_by_id = {}
- version_by_subset_id = {}
- repres_by_version_id = {}
- for subset_id, in_data in asset_entity_data["subsets"].items():
- subset_entity = in_data["subset_entity"]
- subsets_by_id[subset_entity["_id"]] = subset_entity
-
- version_data = in_data["version"]
- version_entity = version_data["version_entity"]
- version_by_subset_id[subset_id] = version_entity
- repres_by_version_id[version_entity["_id"]] = (
- version_data["repres"]
- )
-
- if not subsets_by_id:
- self.log.warning("There are not subsets for asset {0}".format(
- asset_entity["name"]
- ))
- return
-
- profiles_per_subset_id = self._prepare_profile_for_subsets(
- subsets_by_id.values(), valid_profiles
- )
- if not profiles_per_subset_id:
- self.log.warning("There are not valid subsets.")
- return
-
- valid_repres_by_subset_id = collections.defaultdict(list)
- for subset_id, profile in profiles_per_subset_id.items():
- profile_repre_names = profile["repre_names_lowered"]
-
- version_entity = version_by_subset_id[subset_id]
- version_id = version_entity["_id"]
- repres = repres_by_version_id[version_id]
- for repre in repres:
- repre_name_low = repre["name"].lower()
- if repre_name_low in profile_repre_names:
- valid_repres_by_subset_id[subset_id].append(repre)
-
- # DEBUG message
- msg = "Valid representations for Asset: `{}`".format(
- asset_entity["name"]
- )
- for subset_id, repres in valid_repres_by_subset_id.items():
- subset = subsets_by_id[subset_id]
- msg += "\n# Subset Name/ID: `{}`/{}".format(
- subset["name"], subset_id
- )
- for repre in repres:
- msg += "\n## Repre name: `{}`".format(repre["name"])
-
- self.log.debug(msg)
-
- containers = self._load_containers(
- valid_repres_by_subset_id, subsets_by_id,
- profiles_per_subset_id, loaders_by_name
- )
-
- return {
- "asset_entity": asset_entity,
- "containers": containers
- }
-
- @with_avalon
- def _load_containers(
- self, repres_by_subset_id, subsets_by_id,
- profiles_per_subset_id, loaders_by_name
- ):
- """Real load by collected data happens here.
-
- Loading of representations per subset happens here. Each subset can
- loads one representation. Loading is tried in specific order.
- Representations are tried to load by names defined in configuration.
- If subset has representation matching representation name each loader
- is tried to load it until any is successful. If none of them was
- successful then next reprensentation name is tried.
- Subset process loop ends when any representation is loaded or
- all matching representations were already tried.
-
- Args:
- repres_by_subset_id (dict): Available representations mapped
- by their parent (subset) id.
- subsets_by_id (dict): Subset documents mapped by their id.
- profiles_per_subset_id (dict): Build profiles mapped by subset id.
- loaders_by_name (dict): Available loaders per name.
-
- Returns:
- (list) Objects of loaded containers.
- """
- loaded_containers = []
-
- # Get subset id order from build presets.
- build_presets = self.build_presets.get("current_context", [])
- build_presets += self.build_presets.get("linked_assets", [])
- subset_ids_ordered = []
- for preset in build_presets:
- for preset_family in preset["families"]:
- for id, subset in subsets_by_id.items():
- if preset_family not in subset["data"].get("families", []):
- continue
-
- subset_ids_ordered.append(id)
-
- # Order representations from subsets.
- print("repres_by_subset_id", repres_by_subset_id)
- representations_ordered = []
- representations = []
- for id in subset_ids_ordered:
- for subset_id, repres in repres_by_subset_id.items():
- if repres in representations:
- continue
-
- if id == subset_id:
- representations_ordered.append((subset_id, repres))
- representations.append(repres)
-
- print("representations", representations)
-
- # Load ordered reprensentations.
- for subset_id, repres in representations_ordered:
- subset_name = subsets_by_id[subset_id]["name"]
-
- profile = profiles_per_subset_id[subset_id]
- loaders_last_idx = len(profile["loaders"]) - 1
- repre_names_last_idx = len(profile["repre_names_lowered"]) - 1
-
- repre_by_low_name = {
- repre["name"].lower(): repre for repre in repres
- }
-
- is_loaded = False
- for repre_name_idx, profile_repre_name in enumerate(
- profile["repre_names_lowered"]
- ):
- # Break iteration if representation was already loaded
- if is_loaded:
- break
-
- repre = repre_by_low_name.get(profile_repre_name)
- if not repre:
- continue
-
- for loader_idx, loader_name in enumerate(profile["loaders"]):
- if is_loaded:
- break
-
- loader = loaders_by_name.get(loader_name)
- if not loader:
- continue
- try:
- container = avalon.api.load(
- loader,
- repre["_id"],
- name=subset_name
- )
- loaded_containers.append(container)
- is_loaded = True
-
- except Exception as exc:
- if exc == avalon.pipeline.IncompatibleLoaderError:
- self.log.info((
- "Loader `{}` is not compatible with"
- " representation `{}`"
- ).format(loader_name, repre["name"]))
-
- else:
- self.log.error(
- "Unexpected error happened during loading",
- exc_info=True
- )
-
- msg = "Loading failed."
- if loader_idx < loaders_last_idx:
- msg += " Trying next loader."
- elif repre_name_idx < repre_names_last_idx:
- msg += (
- " Loading of subset `{}` was not successful."
- ).format(subset_name)
- else:
- msg += " Trying next representation."
- self.log.info(msg)
-
- return loaded_containers
-
- @with_avalon
- def _collect_last_version_repres(self, asset_entities):
- """Collect subsets, versions and representations for asset_entities.
-
- Args:
- asset_entities (list): Asset entities for which want to find data
-
- Returns:
- (dict): collected entities
-
- Example output:
- ```
- {
- {Asset ID}: {
- "asset_entity": ,
- "subsets": {
- {Subset ID}: {
- "subset_entity": ,
- "version": {
- "version_entity": ,
- "repres": [
- , , ...
- ]
- }
- },
- ...
- }
- },
- ...
- }
- output[asset_id]["subsets"][subset_id]["version"]["repres"]
- ```
- """
-
- if not asset_entities:
- return {}
-
- asset_entity_by_ids = {asset["_id"]: asset for asset in asset_entities}
-
- subsets = list(avalon.io.find({
- "type": "subset",
- "parent": {"$in": asset_entity_by_ids.keys()}
- }))
- subset_entity_by_ids = {subset["_id"]: subset for subset in subsets}
-
- sorted_versions = list(avalon.io.find({
- "type": "version",
- "parent": {"$in": subset_entity_by_ids.keys()}
- }).sort("name", -1))
-
- subset_id_with_latest_version = []
- last_versions_by_id = {}
- for version in sorted_versions:
- subset_id = version["parent"]
- if subset_id in subset_id_with_latest_version:
- continue
- subset_id_with_latest_version.append(subset_id)
- last_versions_by_id[version["_id"]] = version
-
- repres = avalon.io.find({
- "type": "representation",
- "parent": {"$in": last_versions_by_id.keys()}
- })
-
- output = {}
- for repre in repres:
- version_id = repre["parent"]
- version = last_versions_by_id[version_id]
-
- subset_id = version["parent"]
- subset = subset_entity_by_ids[subset_id]
-
- asset_id = subset["parent"]
- asset = asset_entity_by_ids[asset_id]
-
- if asset_id not in output:
- output[asset_id] = {
- "asset_entity": asset,
- "subsets": {}
- }
-
- if subset_id not in output[asset_id]["subsets"]:
- output[asset_id]["subsets"][subset_id] = {
- "subset_entity": subset,
- "version": {
- "version_entity": version,
- "repres": []
- }
- }
-
- output[asset_id]["subsets"][subset_id]["version"]["repres"].append(
- repre
- )
-
- return output
+ return BuildWorkfile()
-@with_avalon
+@deprecated("openpype.pipeline.create.get_legacy_creator_by_name")
def get_creator_by_name(creator_name, case_sensitive=False):
"""Find creator plugin by name.
@@ -1414,30 +738,27 @@ def get_creator_by_name(creator_name, case_sensitive=False):
Returns:
Creator: Return first matching plugin or `None`.
- """
- # Lower input creator name if is not case sensitive
- if not case_sensitive:
- creator_name = creator_name.lower()
-
- for creator_plugin in avalon.api.discover(avalon.api.Creator):
- _creator_name = creator_plugin.__name__
- # Lower creator plugin name if is not case sensitive
- if not case_sensitive:
- _creator_name = _creator_name.lower()
+ Deprecated:
+ Function will be removed after release version 3.16.*
+ """
+ from openpype.pipeline.create import get_legacy_creator_by_name
- if _creator_name == creator_name:
- return creator_plugin
- return None
+ return get_legacy_creator_by_name(creator_name, case_sensitive)
-@with_avalon
+@deprecated
def change_timer_to_current_context():
"""Called after context change to change timers.
- TODO:
- - use TimersManager's static method instead of reimplementing it here
+ Deprecated:
+ This method is specific for TimersManager module so please use the
+ functionality from there. Function will be removed after release
+ version 3.15.*
"""
+
+ from openpype.pipeline import legacy_io
+
webserver_url = os.environ.get("OPENPYPE_WEBSERVER_URL")
if not webserver_url:
log.warning("Couldn't find webserver url")
@@ -1450,9 +771,9 @@ def change_timer_to_current_context():
log.warning("Couldn't start timer")
return
data = {
- "project_name": avalon.io.Session["AVALON_PROJECT"],
- "asset_name": avalon.io.Session["AVALON_ASSET"],
- "task_name": avalon.io.Session["AVALON_TASK"]
+ "project_name": legacy_io.Session["AVALON_PROJECT"],
+ "asset_name": legacy_io.Session["AVALON_ASSET"],
+ "task_name": legacy_io.Session["AVALON_TASK"]
}
requests.post(rest_api_url, json=data)
@@ -1480,6 +801,7 @@ def _get_task_context_data_for_anatomy(
"""
if anatomy is None:
+ from openpype.pipeline import Anatomy
anatomy = Anatomy(project_doc["name"])
asset_name = asset_doc["name"]
@@ -1505,7 +827,7 @@ def _get_task_context_data_for_anatomy(
"requested task type: `{}`".format(task_type)
)
- return {
+ data = {
"project": {
"name": project_doc["name"],
"code": project_doc["data"].get("code")
@@ -1514,11 +836,18 @@ def _get_task_context_data_for_anatomy(
"task": {
"name": task_name,
"type": task_type,
- "short_name": project_task_type_data["short_name"]
+ "short": project_task_type_data["short_name"]
}
}
+ system_general_data = get_system_general_anatomy_data()
+ data.update(system_general_data)
+
+ return data
+
+@deprecated(
+ "openpype.pipeline.workfile.get_custom_workfile_template_by_context")
def get_custom_workfile_template_by_context(
template_profiles, project_doc, asset_doc, task_name, anatomy=None
):
@@ -1540,11 +869,13 @@ def get_custom_workfile_template_by_context(
Returns:
str: Path to template or None if none of profiles match current
context. (Existence of formatted path is not validated.)
- """
- from openpype.lib import filter_profiles
+ Deprecated:
+ Function will be removed after release version 3.16.*
+ """
if anatomy is None:
+ from openpype.pipeline import Anatomy
anatomy = Anatomy(project_doc["name"])
# get project, asset, task anatomy context data
@@ -1560,17 +891,22 @@ def get_custom_workfile_template_by_context(
# get path from matching profile
matching_item = filter_profiles(
template_profiles,
- {"task_type": current_task_type}
+ {"task_types": current_task_type}
)
# when path is available try to format it in case
# there are some anatomy template strings
if matching_item:
template = matching_item["path"][platform.system().lower()]
- return template.format(**anatomy_context_data)
+ return StringTemplate.format_strict_template(
+ template, anatomy_context_data
+ )
return None
+@deprecated(
+ "openpype.pipeline.workfile.get_custom_workfile_template_by_string_context"
+)
def get_custom_workfile_template_by_string_context(
template_profiles, project_name, asset_name, task_name,
dbcon=None, anatomy=None
@@ -1594,47 +930,35 @@ def get_custom_workfile_template_by_string_context(
Returns:
str: Path to template or None if none of profiles match current
context. (Existence of formatted path is not validated.)
+
+ Deprecated:
+ Function will be removed after release version 3.16.*
"""
- if dbcon is None:
- from avalon.api import AvalonMongoDB
+ project_name = None
+ if anatomy is not None:
+ project_name = anatomy.project_name
- dbcon = AvalonMongoDB()
+ if not project_name and dbcon is not None:
+ project_name = dbcon.active_project()
- dbcon.install()
+ if not project_name:
+ raise ValueError("Can't determina project")
- if dbcon.Session["AVALON_PROJECT"] != project_name:
- dbcon.Session["AVALON_PROJECT"] = project_name
-
- project_doc = dbcon.find_one(
- {"type": "project"},
- # All we need is "name" and "data.code" keys
- {
- "name": 1,
- "data.code": 1
- }
- )
- asset_doc = dbcon.find_one(
- {
- "type": "asset",
- "name": asset_name
- },
- # All we need is "name" and "data.tasks" keys
- {
- "name": 1,
- "data.tasks": 1
- }
- )
+ project_doc = get_project(project_name, fields=["name", "data.code"])
+ asset_doc = get_asset_by_name(
+ project_name, asset_name, fields=["name", "data.tasks"])
return get_custom_workfile_template_by_context(
template_profiles, project_doc, asset_doc, task_name, anatomy
)
+@deprecated("openpype.pipeline.context_tools.get_custom_workfile_template")
def get_custom_workfile_template(template_profiles):
"""Filter and fill workfile template profiles by current context.
- Current context is defined by `avalon.api.Session`. That's why this
+ Current context is defined by `legacy_io.Session`. That's why this
function should be used only inside host where context is set and stable.
Args:
@@ -1643,14 +967,111 @@ def get_custom_workfile_template(template_profiles):
Returns:
str: Path to template or None if none of profiles match current
context. (Existence of formatted path is not validated.)
+
+ Deprecated:
+ Function will be removed after release version 3.16.*
"""
- # Use `avalon.io` as Mongo connection
- from avalon import io
+
+ from openpype.pipeline import legacy_io
return get_custom_workfile_template_by_string_context(
template_profiles,
- io.Session["AVALON_PROJECT"],
- io.Session["AVALON_ASSET"],
- io.Session["AVALON_TASK"],
- io
+ legacy_io.Session["AVALON_PROJECT"],
+ legacy_io.Session["AVALON_ASSET"],
+ legacy_io.Session["AVALON_TASK"],
+ legacy_io
+ )
+
+
+@deprecated("openpype.pipeline.workfile.get_last_workfile_with_version")
+def get_last_workfile_with_version(
+ workdir, file_template, fill_data, extensions
+):
+ """Return last workfile version.
+
+ Args:
+ workdir(str): Path to dir where workfiles are stored.
+ file_template(str): Template of file name.
+ fill_data(dict): Data for filling template.
+ extensions(list, tuple): All allowed file extensions of workfile.
+
+ Returns:
+ tuple: Last workfile with version if there is any otherwise
+ returns (None, None).
+
+ Deprecated:
+ Function will be removed after release version 3.16.*
+ """
+
+ from openpype.pipeline.workfile import get_last_workfile_with_version
+
+ return get_last_workfile_with_version(
+ workdir, file_template, fill_data, extensions
+ )
+
+
+@deprecated("openpype.pipeline.workfile.get_last_workfile")
+def get_last_workfile(
+ workdir, file_template, fill_data, extensions, full_path=False
+):
+ """Return last workfile filename.
+
+ Returns file with version 1 if there is not workfile yet.
+
+ Args:
+ workdir(str): Path to dir where workfiles are stored.
+ file_template(str): Template of file name.
+ fill_data(dict): Data for filling template.
+ extensions(list, tuple): All allowed file extensions of workfile.
+ full_path(bool): Full path to file is returned if set to True.
+
+ Returns:
+ str: Last or first workfile as filename of full path to filename.
+
+ Deprecated:
+ Function will be removed after release version 3.16.*
+ """
+
+ from openpype.pipeline.workfile import get_last_workfile
+
+ return get_last_workfile(
+ workdir, file_template, fill_data, extensions, full_path
)
+
+
+@deprecated("openpype.client.get_linked_representation_id")
+def get_linked_ids_for_representations(
+ project_name, repre_ids, dbcon=None, link_type=None, max_depth=0
+):
+ """Returns list of linked ids of particular type (if provided).
+
+ Goes from representations to version, back to representations
+ Args:
+ project_name (str)
+ repre_ids (list) or (ObjectId)
+ dbcon (avalon.mongodb.AvalonMongoDB, optional): Avalon Mongo connection
+ with Session.
+ link_type (str): ['reference', '..]
+ max_depth (int): limit how many levels of recursion
+
+ Returns:
+ (list) of ObjectId - linked representations
+
+ Deprecated:
+ Function will be removed after release version 3.16.*
+ """
+
+ from openpype.client import get_linked_representation_id
+
+ if not isinstance(repre_ids, list):
+ repre_ids = [repre_ids]
+
+ output = []
+ for repre_id in repre_ids:
+ output.extend(get_linked_representation_id(
+ project_name,
+ repre_id=repre_id,
+ link_type=link_type,
+ max_depth=max_depth
+ ))
+ return output
diff --git a/openpype/lib/config.py b/openpype/lib/config.py
index ba394cfd56a..26822649e42 100644
--- a/openpype/lib/config.py
+++ b/openpype/lib/config.py
@@ -1,76 +1,41 @@
-# -*- coding: utf-8 -*-
-"""Get configuration data."""
-import datetime
+import warnings
+import functools
-def get_datetime_data(datetime_obj=None):
- """Returns current datetime data as dictionary.
+class ConfigDeprecatedWarning(DeprecationWarning):
+ pass
- Args:
- datetime_obj (datetime): Specific datetime object
- Returns:
- dict: prepared date & time data
+def deprecated(func):
+ """Mark functions as deprecated.
- Available keys:
- "d" - in shortest possible way.
- "dd" - with 2 digits.
- "ddd" - shortened week day. e.g.: `Mon`, ...
- "dddd" - full name of week day. e.g.: `Monday`, ...
- "m" - in shortest possible way. e.g.: `1` if January
- "mm" - with 2 digits.
- "mmm" - shortened month name. e.g.: `Jan`, ...
- "mmmm" - full month name. e.g.: `January`, ...
- "yy" - shortened year. e.g.: `19`, `20`, ...
- "yyyy" - full year. e.g.: `2019`, `2020`, ...
- "H" - shortened hours.
- "HH" - with 2 digits.
- "h" - shortened hours.
- "hh" - with 2 digits.
- "ht" - AM or PM.
- "M" - shortened minutes.
- "MM" - with 2 digits.
- "S" - shortened seconds.
- "SS" - with 2 digits.
+ It will result in a warning being emitted when the function is used.
"""
- if not datetime_obj:
- datetime_obj = datetime.datetime.now()
-
- year = datetime_obj.strftime("%Y")
+ @functools.wraps(func)
+ def new_func(*args, **kwargs):
+ warnings.simplefilter("always", ConfigDeprecatedWarning)
+ warnings.warn(
+ (
+ "Deprecated import of function '{}'."
+ " Class was moved to 'openpype.lib.dateutils.{}'."
+ " Please change your imports."
+ ).format(func.__name__),
+ category=ConfigDeprecatedWarning
+ )
+ return func(*args, **kwargs)
+ return new_func
+
+
+@deprecated
+def get_datetime_data(datetime_obj=None):
+ from .dateutils import get_datetime_data
- month = datetime_obj.strftime("%m")
- month_name_full = datetime_obj.strftime("%B")
- month_name_short = datetime_obj.strftime("%b")
- day = datetime_obj.strftime("%d")
+ return get_datetime_data(datetime_obj)
- weekday_full = datetime_obj.strftime("%A")
- weekday_short = datetime_obj.strftime("%a")
- hours = datetime_obj.strftime("%H")
- hours_midday = datetime_obj.strftime("%I")
- hour_midday_type = datetime_obj.strftime("%p")
- minutes = datetime_obj.strftime("%M")
- seconds = datetime_obj.strftime("%S")
+@deprecated
+def get_formatted_current_time():
+ from .dateutils import get_formatted_current_time
- return {
- "d": str(int(day)),
- "dd": str(day),
- "ddd": weekday_short,
- "dddd": weekday_full,
- "m": str(int(month)),
- "mm": str(month),
- "mmm": month_name_short,
- "mmmm": month_name_full,
- "yy": str(year[2:]),
- "yyyy": str(year),
- "H": str(int(hours)),
- "HH": str(hours),
- "h": str(int(hours_midday)),
- "hh": str(hours_midday),
- "ht": hour_midday_type,
- "M": str(int(minutes)),
- "MM": str(minutes),
- "S": str(int(seconds)),
- "SS": str(seconds),
- }
+ return get_formatted_current_time()
diff --git a/openpype/lib/connections.py b/openpype/lib/connections.py
new file mode 100644
index 00000000000..91b745a4c1b
--- /dev/null
+++ b/openpype/lib/connections.py
@@ -0,0 +1,38 @@
+import requests
+import os
+
+
+def requests_post(*args, **kwargs):
+ """Wrap request post method.
+
+ Disabling SSL certificate validation if ``DONT_VERIFY_SSL`` environment
+ variable is found. This is useful when Deadline or Muster server are
+ running with self-signed certificates and their certificate is not
+ added to trusted certificates on client machines.
+
+ Warning:
+ Disabling SSL certificate validation is defeating one line
+ of defense SSL is providing and it is not recommended.
+
+ """
+ if "verify" not in kwargs:
+ kwargs["verify"] = not os.getenv("OPENPYPE_DONT_VERIFY_SSL", True)
+ return requests.post(*args, **kwargs)
+
+
+def requests_get(*args, **kwargs):
+ """Wrap request get method.
+
+ Disabling SSL certificate validation if ``DONT_VERIFY_SSL`` environment
+ variable is found. This is useful when Deadline or Muster server are
+ running with self-signed certificates and their certificate is not
+ added to trusted certificates on client machines.
+
+ Warning:
+ Disabling SSL certificate validation is defeating one line
+ of defense SSL is providing and it is not recommended.
+
+ """
+ if "verify" not in kwargs:
+ kwargs["verify"] = not os.getenv("OPENPYPE_DONT_VERIFY_SSL", True)
+ return requests.get(*args, **kwargs)
diff --git a/openpype/lib/dateutils.py b/openpype/lib/dateutils.py
new file mode 100644
index 00000000000..68cd1d1c5bf
--- /dev/null
+++ b/openpype/lib/dateutils.py
@@ -0,0 +1,95 @@
+# -*- coding: utf-8 -*-
+"""Get configuration data."""
+import datetime
+
+
+def get_datetime_data(datetime_obj=None):
+ """Returns current datetime data as dictionary.
+
+ Args:
+ datetime_obj (datetime): Specific datetime object
+
+ Returns:
+ dict: prepared date & time data
+
+ Available keys:
+ "d" - in shortest possible way.
+ "dd" - with 2 digits.
+ "ddd" - shortened week day. e.g.: `Mon`, ...
+ "dddd" - full name of week day. e.g.: `Monday`, ...
+ "m" - in shortest possible way. e.g.: `1` if January
+ "mm" - with 2 digits.
+ "mmm" - shortened month name. e.g.: `Jan`, ...
+ "mmmm" - full month name. e.g.: `January`, ...
+ "yy" - shortened year. e.g.: `19`, `20`, ...
+ "yyyy" - full year. e.g.: `2019`, `2020`, ...
+ "H" - shortened hours.
+ "HH" - with 2 digits.
+ "h" - shortened hours.
+ "hh" - with 2 digits.
+ "ht" - AM or PM.
+ "M" - shortened minutes.
+ "MM" - with 2 digits.
+ "S" - shortened seconds.
+ "SS" - with 2 digits.
+ """
+
+ if not datetime_obj:
+ datetime_obj = datetime.datetime.now()
+
+ year = datetime_obj.strftime("%Y")
+
+ month = datetime_obj.strftime("%m")
+ month_name_full = datetime_obj.strftime("%B")
+ month_name_short = datetime_obj.strftime("%b")
+ day = datetime_obj.strftime("%d")
+
+ weekday_full = datetime_obj.strftime("%A")
+ weekday_short = datetime_obj.strftime("%a")
+
+ hours = datetime_obj.strftime("%H")
+ hours_midday = datetime_obj.strftime("%I")
+ hour_midday_type = datetime_obj.strftime("%p")
+ minutes = datetime_obj.strftime("%M")
+ seconds = datetime_obj.strftime("%S")
+
+ return {
+ "d": str(int(day)),
+ "dd": str(day),
+ "ddd": weekday_short,
+ "dddd": weekday_full,
+ "m": str(int(month)),
+ "mm": str(month),
+ "mmm": month_name_short,
+ "mmmm": month_name_full,
+ "yy": str(year[2:]),
+ "yyyy": str(year),
+ "H": str(int(hours)),
+ "HH": str(hours),
+ "h": str(int(hours_midday)),
+ "hh": str(hours_midday),
+ "ht": hour_midday_type,
+ "M": str(int(minutes)),
+ "MM": str(minutes),
+ "S": str(int(seconds)),
+ "SS": str(seconds),
+ }
+
+
+def get_timestamp(datetime_obj=None):
+ """Get standardized timestamp from datetime object.
+
+ Args:
+ datetime_obj (datetime.datetime): Object of datetime. Current time
+ is used if not passed.
+ """
+
+ if datetime_obj is None:
+ datetime_obj = datetime.datetime.now()
+ return datetime_obj.strftime(
+ "%Y%m%dT%H%M%SZ"
+ )
+
+
+def get_formatted_current_time():
+ return get_timestamp()
diff --git a/openpype/lib/delivery.py b/openpype/lib/delivery.py
index 01fcc907ed0..efb542de753 100644
--- a/openpype/lib/delivery.py
+++ b/openpype/lib/delivery.py
@@ -1,215 +1,218 @@
"""Functions useful for delivery action or loader"""
import os
import shutil
-import glob
-import clique
-import collections
+import functools
+import warnings
-def collect_frames(files):
+class DeliveryDeprecatedWarning(DeprecationWarning):
+ pass
+
+
+def deprecated(new_destination):
+ """Mark functions as deprecated.
+
+ It will result in a warning being emitted when the function is used.
"""
- Returns dict of source path and its frame, if from sequence
- Uses clique as most precise solution
+ func = None
+ if callable(new_destination):
+ func = new_destination
+ new_destination = None
+
+ def _decorator(decorated_func):
+ if new_destination is None:
+ warning_message = (
+ " Please check content of deprecated function to figure out"
+ " possible replacement."
+ )
+ else:
+ warning_message = " Please replace your usage with '{}'.".format(
+ new_destination
+ )
+
+ @functools.wraps(decorated_func)
+ def wrapper(*args, **kwargs):
+ warnings.simplefilter("always", DeliveryDeprecatedWarning)
+ warnings.warn(
+ (
+ "Call to deprecated function '{}'"
+ "\nFunction was moved or removed.{}"
+ ).format(decorated_func.__name__, warning_message),
+ category=DeliveryDeprecatedWarning,
+ stacklevel=4
+ )
+ return decorated_func(*args, **kwargs)
+ return wrapper
+
+ if func is None:
+ return _decorator
+ return _decorator(func)
+
+
+@deprecated("openpype.lib.path_tools.collect_frames")
+def collect_frames(files):
+ """Returns dict of source path and its frame, if from sequence
- Args:
- files(list): list of source paths
- Returns:
- (dict): {'/asset/subset_v001.0001.png': '0001', ....}
+ Uses clique as most precise solution, used when anatomy template that
+ created files is not known.
+
+ Assumption is that frames are separated by '.', negative frames are not
+ allowed.
+
+ Args:
+ files(list) or (set with single value): list of source paths
+
+ Returns:
+ (dict): {'/asset/subset_v001.0001.png': '0001', ....}
+
+ Deprecated:
+ Function was moved to different location and will be removed
+ after 3.16.* release.
"""
- collections, remainder = clique.assemble(files)
- sources_and_frames = {}
- if collections:
- for collection in collections:
- src_head = collection.head
- src_tail = collection.tail
+ from .path_tools import collect_frames
- for index in collection.indexes:
- src_frame = collection.format("{padding}") % index
- src_file_name = "{}{}{}".format(src_head, src_frame,
- src_tail)
- sources_and_frames[src_file_name] = src_frame
- else:
- sources_and_frames[remainder.pop()] = None
+ return collect_frames(files)
- return sources_and_frames
+@deprecated("openpype.lib.path_tools.format_file_size")
+def sizeof_fmt(num, suffix=None):
+ """Returns formatted string with size in appropriate unit
-def sizeof_fmt(num, suffix='B'):
- """Returns formatted string with size in appropriate unit"""
- for unit in ['', 'Ki', 'Mi', 'Gi', 'Ti', 'Pi', 'Ei', 'Zi']:
- if abs(num) < 1024.0:
- return "%3.1f%s%s" % (num, unit, suffix)
- num /= 1024.0
- return "%.1f%s%s" % (num, 'Yi', suffix)
+ Deprecated:
+ Function was moved to different location and will be removed
+ after 3.16.* release.
+ """
+ from .path_tools import format_file_size
+ return format_file_size(num, suffix)
-def path_from_representation(representation, anatomy):
- from avalon import pipeline # safer importing
- try:
- template = representation["data"]["template"]
+@deprecated("openpype.pipeline.load.get_representation_path_with_anatomy")
+def path_from_representation(representation, anatomy):
+ """Get representation path using representation document and anatomy.
- except KeyError:
- return None
+ Args:
+ representation (Dict[str, Any]): Representation document.
+ anatomy (Anatomy): Project anatomy.
- try:
- context = representation["context"]
- context["root"] = anatomy.roots
- path = pipeline.format_template_with_optional_keys(
- context, template
- )
- path = os.path.normpath(path.replace("/", "\\"))
+ Deprecated:
+ Function was moved to different location and will be removed
+ after 3.16.* release.
+ """
- except KeyError:
- # Template references unavailable data
- return None
+ from openpype.pipeline.load import get_representation_path_with_anatomy
- return path
+ return get_representation_path_with_anatomy(representation, anatomy)
+@deprecated
def copy_file(src_path, dst_path):
"""Hardlink file if possible(to save space), copy if not"""
- from avalon.vendor import filelink # safer importing
+ from openpype.lib import create_hard_link # safer importing
if os.path.exists(dst_path):
return
try:
- filelink.create(
+ create_hard_link(
src_path,
- dst_path,
- filelink.HARDLINK
+ dst_path
)
except OSError:
shutil.copyfile(src_path, dst_path)
+@deprecated("openpype.pipeline.delivery.get_format_dict")
def get_format_dict(anatomy, location_path):
"""Returns replaced root values from user provider value.
- Args:
- anatomy (Anatomy)
- location_path (str): user provided value
- Returns:
- (dict): prepared for formatting of a template
+ Args:
+ anatomy (Anatomy)
+ location_path (str): user provided value
+
+ Returns:
+ (dict): prepared for formatting of a template
+
+ Deprecated:
+ Function was moved to different location and will be removed
+ after 3.16.* release.
"""
- format_dict = {}
- if location_path:
- location_path = location_path.replace("\\", "/")
- root_names = anatomy.root_names_from_templates(
- anatomy.templates["delivery"]
- )
- if root_names is None:
- format_dict["root"] = location_path
- else:
- format_dict["root"] = {}
- for name in root_names:
- format_dict["root"][name] = location_path
- return format_dict
+ from openpype.pipeline.delivery import get_format_dict
+ return get_format_dict(anatomy, location_path)
+
+
+@deprecated("openpype.pipeline.delivery.check_destination_path")
def check_destination_path(repre_id,
anatomy, anatomy_data,
datetime_data, template_name):
""" Try to create destination path based on 'template_name'.
- In the case that path cannot be filled, template contains unmatched
- keys, provide error message to filter out repre later.
-
- Args:
- anatomy (Anatomy)
- anatomy_data (dict): context to fill anatomy
- datetime_data (dict): values with actual date
- template_name (str): to pick correct delivery template
- Returns:
- (collections.defauldict): {"TYPE_OF_ERROR":"ERROR_DETAIL"}
- """
- anatomy_data.update(datetime_data)
- anatomy_filled = anatomy.format_all(anatomy_data)
- dest_path = anatomy_filled["delivery"][template_name]
- report_items = collections.defaultdict(list)
-
- if not dest_path.solved:
- msg = (
- "Missing keys in Representation's context"
- " for anatomy template \"{}\"."
- ).format(template_name)
-
- sub_msg = (
- "Representation: {}")
+ ```
+
+ Args:
+ *args (tuple): OpenPype cli arguments.
+ **kwargs (dict): Keyword arguments for for subprocess.Popen.
+ """
+ args = get_openpype_execute_args(*args)
+ env = kwargs.pop("env", None)
+ # Keep env untouched if are passed and not empty
+ if not env:
+ # Skip envs that can affect OpenPype process
+ # - fill more if you find more
+ env = clean_envs_for_openpype_process(os.environ)
+ return run_subprocess(args, env=env, **kwargs)
+
+
+def run_detached_process(args, **kwargs):
+ """Execute process with passed arguments as separated process.
+
+ Values from 'os.environ' are used for environments if are not passed.
+ They are cleaned using 'clean_envs_for_openpype_process' function.
+
+ Example:
+ ```
+ run_detached_openpype_process("run", "")
+ ```
+
+ Args:
+ *args (tuple): OpenPype cli arguments.
+ **kwargs (dict): Keyword arguments for for subprocess.Popen.
+
+ Returns:
+ subprocess.Popen: Pointer to launched process but it is possible that
+ launched process is already killed (on linux).
+ """
+ env = kwargs.pop("env", None)
+ # Keep env untouched if are passed and not empty
+ if not env:
+ env = os.environ
+
+ # Create copy of passed env
+ kwargs["env"] = {k: v for k, v in env.items()}
+
+ low_platform = platform.system().lower()
+ if low_platform == "darwin":
+ new_args = ["open", "-na", args.pop(0), "--args"]
+ new_args.extend(args)
+ args = new_args
+
+ elif low_platform == "windows":
+ flags = (
+ subprocess.CREATE_NEW_PROCESS_GROUP
+ | subprocess.DETACHED_PROCESS
+ )
+ kwargs["creationflags"] = flags
+
+ if not sys.stdout:
+ kwargs["stdout"] = subprocess.DEVNULL
+ kwargs["stderr"] = subprocess.DEVNULL
+
+ elif low_platform == "linux" and get_linux_launcher_args() is not None:
+ json_data = {
+ "args": args,
+ "env": kwargs.pop("env")
+ }
+ json_temp = tempfile.NamedTemporaryFile(
+ mode="w", prefix="op_app_args", suffix=".json", delete=False
+ )
+ json_temp.close()
+ json_temp_filpath = json_temp.name
+ with open(json_temp_filpath, "w") as stream:
+ json.dump(json_data, stream)
+
+ new_args = get_linux_launcher_args()
+ new_args.append(json_temp_filpath)
+
+ # Create mid-process which will launch application
+ process = subprocess.Popen(new_args, **kwargs)
+ # Wait until the process finishes
+ # - This is important! The process would stay in "open" state.
+ process.wait()
+ # Remove the temp file
+ os.remove(json_temp_filpath)
+ # Return process which is already terminated
+ return process
+
+ process = subprocess.Popen(args, **kwargs)
+ return process
+
+
def path_to_subprocess_arg(path):
"""Prepare path for subprocess arguments.
@@ -148,6 +268,18 @@ def path_to_subprocess_arg(path):
def get_pype_execute_args(*args):
+ """Backwards compatible function for 'get_openpype_execute_args'."""
+ import traceback
+
+ log = Logger.get_logger("get_pype_execute_args")
+ stack = "\n".join(traceback.format_stack())
+ log.warning((
+ "Using deprecated function 'get_pype_execute_args'. Called from:\n{}"
+ ).format(stack))
+ return get_openpype_execute_args(*args)
+
+
+def get_openpype_execute_args(*args):
"""Arguments to run pype command.
Arguments for subprocess when need to spawn new pype process. Which may be
@@ -175,3 +307,46 @@ def get_pype_execute_args(*args):
pype_args.extend(args)
return pype_args
+
+
+def get_linux_launcher_args(*args):
+ """Path to application mid process executable.
+
+ This function should be able as arguments are different when used
+ from code and build.
+
+ It is possible that this function is used in OpenPype build which does
+ not have yet the new executable. In that case 'None' is returned.
+
+ Args:
+ args (iterable): List of additional arguments added after executable
+ argument.
+
+ Returns:
+ list: Executables with possible positional argument to script when
+ called from code.
+ """
+ filename = "app_launcher"
+ openpype_executable = os.environ["OPENPYPE_EXECUTABLE"]
+
+ executable_filename = os.path.basename(openpype_executable)
+ if "python" in executable_filename.lower():
+ script_path = os.path.join(
+ os.environ["OPENPYPE_ROOT"],
+ "{}.py".format(filename)
+ )
+ launch_args = [openpype_executable, script_path]
+ else:
+ new_executable = os.path.join(
+ os.path.dirname(openpype_executable),
+ filename
+ )
+ executable_path = find_executable(new_executable)
+ if executable_path is None:
+ return None
+ launch_args = [executable_path]
+
+ if args:
+ launch_args.extend(args)
+
+ return launch_args
diff --git a/openpype/lib/file_transaction.py b/openpype/lib/file_transaction.py
new file mode 100644
index 00000000000..1626bec6b66
--- /dev/null
+++ b/openpype/lib/file_transaction.py
@@ -0,0 +1,171 @@
+import os
+import logging
+import sys
+import errno
+import six
+
+from openpype.lib import create_hard_link
+
+# this is needed until speedcopy for linux is fixed
+if sys.platform == "win32":
+ from speedcopy import copyfile
+else:
+ from shutil import copyfile
+
+
+class FileTransaction(object):
+ """
+
+ The file transaction is a three step process.
+
+ 1) Rename any existing files to a "temporary backup" during `process()`
+ 2) Copy the files to final destination during `process()`
+ 3) Remove any backed up files (*no rollback possible!) during `finalize()`
+
+ Step 3 is done during `finalize()`. If not called the .bak files will
+ remain on disk.
+
+ These steps try to ensure that we don't overwrite half of any existing
+ files e.g. if they are currently in use.
+
+ Note:
+ A regular filesystem is *not* a transactional file system and even
+ though this implementation tries to produce a 'safe copy' with a
+ potential rollback do keep in mind that it's inherently unsafe due
+ to how filesystem works and a myriad of things could happen during
+ the transaction that break the logic. A file storage could go down,
+ permissions could be changed, other machines could be moving or writing
+ files. A lot can happen.
+
+ Warning:
+ Any folders created during the transfer will not be removed.
+
+ """
+
+ MODE_COPY = 0
+ MODE_HARDLINK = 1
+
+ def __init__(self, log=None):
+
+ if log is None:
+ log = logging.getLogger("FileTransaction")
+
+ self.log = log
+
+ # The transfer queue
+ # todo: make this an actual FIFO queue?
+ self._transfers = {}
+
+ # Destination file paths that a file was transferred to
+ self._transferred = []
+
+ # Backup file location mapping to original locations
+ self._backup_to_original = {}
+
+ def add(self, src, dst, mode=MODE_COPY):
+ """Add a new file to transfer queue"""
+ opts = {"mode": mode}
+
+ src = os.path.abspath(src)
+ dst = os.path.abspath(dst)
+
+ if dst in self._transfers:
+ queued_src = self._transfers[dst][0]
+ if src == queued_src:
+ self.log.debug("File transfer was already "
+ "in queue: {} -> {}".format(src, dst))
+ return
+ else:
+ self.log.warning("File transfer in queue replaced..")
+ self.log.debug("Removed from queue: "
+ "{} -> {}".format(queued_src, dst))
+ self.log.debug("Added to queue: {} -> {}".format(src, dst))
+
+ self._transfers[dst] = (src, opts)
+
+ def process(self):
+
+ # Backup any existing files
+ for dst in self._transfers.keys():
+ if os.path.exists(dst):
+ # Backup original file
+ # todo: add timestamp or uuid to ensure unique
+ backup = dst + ".bak"
+ self._backup_to_original[backup] = dst
+ self.log.debug("Backup existing file: "
+ "{} -> {}".format(dst, backup))
+ os.rename(dst, backup)
+
+ # Copy the files to transfer
+ for dst, (src, opts) in self._transfers.items():
+ self._create_folder_for_file(dst)
+
+ if opts["mode"] == self.MODE_COPY:
+ self.log.debug("Copying file ... {} -> {}".format(src, dst))
+ copyfile(src, dst)
+ elif opts["mode"] == self.MODE_HARDLINK:
+ self.log.debug("Hardlinking file ... {} -> {}".format(src,
+ dst))
+ create_hard_link(src, dst)
+
+ self._transferred.append(dst)
+
+ def finalize(self):
+ # Delete any backed up files
+ for backup in self._backup_to_original.keys():
+ try:
+ os.remove(backup)
+ except OSError:
+ self.log.error("Failed to remove backup file: "
+ "{}".format(backup),
+ exc_info=True)
+
+ def rollback(self):
+
+ errors = 0
+
+ # Rollback any transferred files
+ for path in self._transferred:
+ try:
+ os.remove(path)
+ except OSError:
+ errors += 1
+ self.log.error("Failed to rollback created file: "
+ "{}".format(path),
+ exc_info=True)
+
+ # Rollback the backups
+ for backup, original in self._backup_to_original.items():
+ try:
+ os.rename(backup, original)
+ except OSError:
+ errors += 1
+ self.log.error("Failed to restore original file: "
+ "{} -> {}".format(backup, original),
+ exc_info=True)
+
+ if errors:
+ self.log.error("{} errors occurred during "
+ "rollback.".format(errors), exc_info=True)
+ six.reraise(*sys.exc_info())
+
+ @property
+ def transferred(self):
+ """Return the processed transfers destination paths"""
+ return list(self._transferred)
+
+ @property
+ def backups(self):
+ """Return the backup file paths"""
+ return list(self._backup_to_original.keys())
+
+ def _create_folder_for_file(self, path):
+ dirname = os.path.dirname(path)
+ try:
+ os.makedirs(dirname)
+ except OSError as e:
+ if e.errno == errno.EEXIST:
+ pass
+ else:
+ self.log.critical("An unexpected error occurred.")
+ six.reraise(*sys.exc_info())
diff --git a/openpype/lib/git_progress.py b/openpype/lib/git_progress.py
deleted file mode 100644
index e9cf9a12e1f..00000000000
--- a/openpype/lib/git_progress.py
+++ /dev/null
@@ -1,86 +0,0 @@
-import git
-from tqdm import tqdm
-
-
-class _GitProgress(git.remote.RemoteProgress):
- """ Class handling displaying progress during git operations.
-
- This is using **tqdm** for showing progress bars. As **GitPython**
- is parsing progress directly from git command, it is somehow unreliable
- as in some operations it is difficult to get total count of iterations
- to display meaningful progress bar.
-
- """
- _t = None
- _code = 0
- _current_status = ''
- _current_max = ''
-
- _description = {
- 256: "Checking out files",
- 4: "Counting objects",
- 128: "Finding sources",
- 32: "Receiving objects",
- 64: "Resolving deltas",
- 16: "Writing objects"
- }
-
- def __init__(self):
- super().__init__()
-
- def __del__(self):
- if self._t is not None:
- self._t.close()
-
- def _detroy_tqdm(self):
- """ Used to close tqdm when opration ended.
-
- """
- if self._t is not None:
- self._t.close()
- self._t = None
-
- def _check_mask(self, opcode: int) -> bool:
- """" Add meaningful description to **GitPython** opcodes.
-
- :param opcode: OP_MASK opcode
- :type opcode: int
- :return: String description of opcode
- :rtype: str
-
- .. seealso:: For opcodes look at :class:`git.RemoteProgress`
-
- """
- if opcode & self.COUNTING:
- return self._description.get(self.COUNTING)
- elif opcode & self.CHECKING_OUT:
- return self._description.get(self.CHECKING_OUT)
- elif opcode & self.WRITING:
- return self._description.get(self.WRITING)
- elif opcode & self.RECEIVING:
- return self._description.get(self.RECEIVING)
- elif opcode & self.RESOLVING:
- return self._description.get(self.RESOLVING)
- elif opcode & self.FINDING_SOURCES:
- return self._description.get(self.FINDING_SOURCES)
- else:
- return "Processing"
-
- def update(self, op_code, cur_count, max_count=None, message=''):
- """ Called when git operation update progress.
-
- .. seealso:: For more details see
- :func:`git.objects.submodule.base.Submodule.update`
- `Documentation `_
-
- """
- code = self._check_mask(op_code)
- if self._current_status != code or self._current_max != max_count:
- self._current_max = max_count
- self._current_status = code
- self._detroy_tqdm()
- self._t = tqdm(total=max_count)
- self._t.set_description(" . {}".format(code))
-
- self._t.update(cur_count)
diff --git a/openpype/lib/import_utils.py b/openpype/lib/import_utils.py
deleted file mode 100644
index 4e726188036..00000000000
--- a/openpype/lib/import_utils.py
+++ /dev/null
@@ -1,25 +0,0 @@
-import os
-import sys
-import importlib
-from .log import PypeLogger as Logger
-
-log = Logger().get_logger(__name__)
-
-
-def discover_host_vendor_module(module_name):
- host = os.environ["AVALON_APP"]
- pype_root = os.environ["OPENPYPE_REPOS_ROOT"]
- main_module = module_name.split(".")[0]
- module_path = os.path.join(
- pype_root, "hosts", host, "vendor", main_module)
-
- log.debug(
- "Importing moduel from host vendor path: `{}`".format(module_path))
-
- if not os.path.exists(module_path):
- log.warning(
- "Path not existing: `{}`".format(module_path))
- return None
-
- sys.path.insert(1, module_path)
- return importlib.import_module(module_name)
diff --git a/openpype/lib/local_settings.py b/openpype/lib/local_settings.py
index 97e99b4b5ae..c6c96992409 100644
--- a/openpype/lib/local_settings.py
+++ b/openpype/lib/local_settings.py
@@ -34,7 +34,7 @@ def wrapper(*args, **kwargs):
get_system_settings
)
-from .import validate_mongo_connection
+from openpype.client.mongo import validate_mongo_connection
_PLACEHOLDER = object()
diff --git a/openpype/lib/log.py b/openpype/lib/log.py
index a42faef0081..26dcd86eecc 100644
--- a/openpype/lib/log.py
+++ b/openpype/lib/log.py
@@ -24,12 +24,13 @@
import threading
import copy
-from . import Terminal
-from .mongo import (
+from openpype.client.mongo import (
MongoEnvNotSet,
get_default_components,
- OpenPypeMongoConnection
+ OpenPypeMongoConnection,
)
+from . import Terminal
+
try:
import log4mongo
from log4mongo.handlers import MongoHandler
@@ -41,13 +42,13 @@
USE_UNICODE = hasattr(__builtins__, "unicode")
-class PypeStreamHandler(logging.StreamHandler):
+class LogStreamHandler(logging.StreamHandler):
""" StreamHandler class designed to handle utf errors in python 2.x hosts.
"""
def __init__(self, stream=None):
- super(PypeStreamHandler, self).__init__(stream)
+ super(LogStreamHandler, self).__init__(stream)
self.enabled = True
def enable(self):
@@ -56,7 +57,6 @@ def enable(self):
Used to silence output
"""
self.enabled = True
- pass
def disable(self):
""" Disable StreamHandler
@@ -98,18 +98,22 @@ def emit(self, record):
self.flush()
except (KeyboardInterrupt, SystemExit):
raise
+
+ except OSError:
+ self.handleError(record)
+
except Exception:
print(repr(record))
self.handleError(record)
-class PypeFormatter(logging.Formatter):
+class LogFormatter(logging.Formatter):
DFT = '%(levelname)s >>> { %(name)s }: [ %(message)s ]'
default_formatter = logging.Formatter(DFT)
def __init__(self, formats):
- super(PypeFormatter, self).__init__()
+ super(LogFormatter, self).__init__()
self.formatters = {}
for loglevel in formats:
self.formatters[loglevel] = logging.Formatter(formats[loglevel])
@@ -137,7 +141,7 @@ def format(self, record):
return out
-class PypeMongoFormatter(logging.Formatter):
+class MongoFormatter(logging.Formatter):
DEFAULT_PROPERTIES = logging.LogRecord(
'', '', '', '', '', '', '', '').__dict__.keys()
@@ -157,7 +161,7 @@ def format(self, record):
'method': record.funcName,
'lineNumber': record.lineno
}
- document.update(PypeLogger.get_process_data())
+ document.update(Logger.get_process_data())
# Standard document decorated with exception info
if record.exc_info is not None:
@@ -177,7 +181,7 @@ def format(self, record):
return document
-class PypeLogger:
+class Logger:
DFT = '%(levelname)s >>> { %(name)s }: [ %(message)s ] '
DBG = " - { %(name)s }: [ %(message)s ] "
INF = ">>> [ %(message)s ] "
@@ -212,8 +216,8 @@ class PypeLogger:
# Collection name under database in Mongo
log_collection_name = "logs"
- # OPENPYPE_DEBUG
- pype_debug = 0
+ # Logging level - OPENPYPE_LOG_LEVEL
+ log_level = None
# Data same for all record documents
process_data = None
@@ -227,10 +231,7 @@ def get_logger(cls, name=None, _host=None):
logger = logging.getLogger(name or "__main__")
- if cls.pype_debug > 1:
- logger.setLevel(logging.DEBUG)
- else:
- logger.setLevel(logging.INFO)
+ logger.setLevel(cls.log_level)
add_mongo_handler = cls.use_mongo_logging
add_console_handler = True
@@ -238,7 +239,7 @@ def get_logger(cls, name=None, _host=None):
for handler in logger.handlers:
if isinstance(handler, MongoHandler):
add_mongo_handler = False
- elif isinstance(handler, PypeStreamHandler):
+ elif isinstance(handler, LogStreamHandler):
add_console_handler = False
if add_console_handler:
@@ -291,7 +292,7 @@ def _get_mongo_handler(cls):
"username": components["username"],
"password": components["password"],
"capped": True,
- "formatter": PypeMongoFormatter()
+ "formatter": MongoFormatter()
}
if components["port"] is not None:
kwargs["port"] = int(components["port"])
@@ -302,10 +303,10 @@ def _get_mongo_handler(cls):
@classmethod
def _get_console_handler(cls):
- formatter = PypeFormatter(cls.FORMAT_FILE)
- console_handler = PypeStreamHandler()
+ formatter = LogFormatter(cls.FORMAT_FILE)
+ console_handler = LogStreamHandler()
- console_handler.set_name("PypeStreamHandler")
+ console_handler.set_name("LogStreamHandler")
console_handler.setFormatter(formatter)
return console_handler
@@ -329,6 +330,9 @@ def _initialize(cls):
# Define if should logging to mongo be used
use_mongo_logging = bool(log4mongo is not None)
+ if use_mongo_logging:
+ use_mongo_logging = os.environ.get("OPENPYPE_LOG_TO_SERVER") == "1"
+
# Set mongo id for process (ONLY ONCE)
if use_mongo_logging and cls.mongo_process_id is None:
try:
@@ -353,8 +357,16 @@ def _initialize(cls):
# Store result to class definition
cls.use_mongo_logging = use_mongo_logging
- # Define if is in OPENPYPE_DEBUG mode
- cls.pype_debug = int(os.getenv("OPENPYPE_DEBUG") or "0")
+ # Define what is logging level
+ log_level = os.getenv("OPENPYPE_LOG_LEVEL")
+ if not log_level:
+ # Check OPENPYPE_DEBUG for backwards compatibility
+ op_debug = os.getenv("OPENPYPE_DEBUG")
+ if op_debug and int(op_debug) > 0:
+ log_level = 10
+ else:
+ log_level = 20
+ cls.log_level = int(log_level)
if not os.environ.get("OPENPYPE_MONGO"):
cls.use_mongo_logging = False
@@ -405,9 +417,9 @@ def set_process_name(cls, process_name):
def get_process_name(cls):
"""Process name that is like "label" of a process.
- Pype's logging can be used from pype itseld of from hosts. Even in Pype
- it's good to know if logs are from Pype tray or from pype's event
- server. This should help to identify that information.
+ OpenPype's logging can be used from OpenPyppe itself of from hosts.
+ Even in OpenPype process it's good to know if logs are from tray or
+ from other cli commands. This should help to identify that information.
"""
if cls._process_name is not None:
return cls._process_name
@@ -473,23 +485,19 @@ def get_log_mongo_connection(cls):
return OpenPypeMongoConnection.get_mongo_client()
-def timeit(method):
- """Print time in function.
-
- For debugging.
+class PypeLogger(Logger):
+ """Duplicate of 'Logger'.
+ Deprecated:
+ Class will be removed after release version 3.16.*
"""
- log = logging.getLogger()
-
- def timed(*args, **kw):
- ts = time.time()
- result = method(*args, **kw)
- te = time.time()
- if 'log_time' in kw:
- name = kw.get('log_name', method.__name__.upper())
- kw['log_time'][name] = int((te - ts) * 1000)
- else:
- log.debug('%r %2.2f ms' % (method.__name__, (te - ts) * 1000))
- print('%r %2.2f ms' % (method.__name__, (te - ts) * 1000))
- return result
- return timed
+
+ @classmethod
+ def get_logger(cls, *args, **kwargs):
+ logger = Logger.get_logger(*args, **kwargs)
+ # TODO uncomment when replaced most of places
+ logger.warning((
+ "'openpype.lib.PypeLogger' is deprecated class."
+ " Please use 'openpype.lib.Logger' instead."
+ ))
+ return logger
diff --git a/openpype/lib/mongo.py b/openpype/lib/mongo.py
index 7e0bd4f7969..bb2ee6016a9 100644
--- a/openpype/lib/mongo.py
+++ b/openpype/lib/mongo.py
@@ -1,206 +1,61 @@
-import os
-import sys
-import time
-import logging
-import pymongo
-import certifi
+import warnings
+import functools
+from openpype.client.mongo import (
+ MongoEnvNotSet,
+ OpenPypeMongoConnection,
+)
-if sys.version_info[0] == 2:
- from urlparse import urlparse, parse_qs
-else:
- from urllib.parse import urlparse, parse_qs
-
-class MongoEnvNotSet(Exception):
+class MongoDeprecatedWarning(DeprecationWarning):
pass
-def _decompose_url(url):
- """Decompose mongo url to basic components.
+def mongo_deprecated(func):
+ """Mark functions as deprecated.
- Used for creation of MongoHandler which expect mongo url components as
- separated kwargs. Components are at the end not used as we're setting
- connection directly this is just a dumb components for MongoHandler
- validation pass.
+ It will result in a warning being emitted when the function is used.
"""
- # Use first url from passed url
- # - this is beacuse it is possible to pass multiple urls for multiple
- # replica sets which would crash on urlparse otherwise
- # - please don't use comma in username of password
- url = url.split(",")[0]
- components = {
- "scheme": None,
- "host": None,
- "port": None,
- "username": None,
- "password": None,
- "auth_db": None
- }
-
- result = urlparse(url)
- if result.scheme is None:
- _url = "mongodb://{}".format(url)
- result = urlparse(_url)
-
- components["scheme"] = result.scheme
- components["host"] = result.hostname
- try:
- components["port"] = result.port
- except ValueError:
- raise RuntimeError("invalid port specified")
- components["username"] = result.username
- components["password"] = result.password
-
- try:
- components["auth_db"] = parse_qs(result.query)['authSource'][0]
- except KeyError:
- # no auth db provided, mongo will use the one we are connecting to
- pass
-
- return components
-
-def get_default_components():
- mongo_url = os.environ.get("OPENPYPE_MONGO")
- if mongo_url is None:
- raise MongoEnvNotSet(
- "URL for Mongo logging connection is not set."
+ @functools.wraps(func)
+ def new_func(*args, **kwargs):
+ warnings.simplefilter("always", MongoDeprecatedWarning)
+ warnings.warn(
+ (
+ "Call to deprecated function '{}'."
+ " Function was moved to 'openpype.client.mongo'."
+ ).format(func.__name__),
+ category=MongoDeprecatedWarning,
+ stacklevel=2
)
- return _decompose_url(mongo_url)
+ return func(*args, **kwargs)
+ return new_func
-def should_add_certificate_path_to_mongo_url(mongo_url):
- """Check if should add ca certificate to mongo url.
+@mongo_deprecated
+def get_default_components():
+ from openpype.client.mongo import get_default_components
- Since 30.9.2021 cloud mongo requires newer certificates that are not
- available on most of workstation. This adds path to certifi certificate
- which is valid for it. To add the certificate path url must have scheme
- 'mongodb+srv' or has 'ssl=true' or 'tls=true' in url query.
- """
- parsed = urlparse(mongo_url)
- query = parse_qs(parsed.query)
- lowered_query_keys = set(key.lower() for key in query.keys())
- add_certificate = False
- # Check if url 'ssl' or 'tls' are set to 'true'
- for key in ("ssl", "tls"):
- if key in query and "true" in query["ssl"]:
- add_certificate = True
- break
+ return get_default_components()
- # Check if url contains 'mongodb+srv'
- if not add_certificate and parsed.scheme == "mongodb+srv":
- add_certificate = True
- # Check if url does already contain certificate path
- if add_certificate and "tlscafile" in lowered_query_keys:
- add_certificate = False
+@mongo_deprecated
+def should_add_certificate_path_to_mongo_url(mongo_url):
+ from openpype.client.mongo import should_add_certificate_path_to_mongo_url
- return add_certificate
+ return should_add_certificate_path_to_mongo_url(mongo_url)
+@mongo_deprecated
def validate_mongo_connection(mongo_uri):
- """Check if provided mongodb URL is valid.
-
- Args:
- mongo_uri (str): URL to validate.
-
- Raises:
- ValueError: When port in mongo uri is not valid.
- pymongo.errors.InvalidURI: If passed mongo is invalid.
- pymongo.errors.ServerSelectionTimeoutError: If connection timeout
- passed so probably couldn't connect to mongo server.
-
- """
- client = OpenPypeMongoConnection.create_connection(
- mongo_uri, retry_attempts=1
- )
- client.close()
+ from openpype.client.mongo import validate_mongo_connection
+ return validate_mongo_connection(mongo_uri)
-class OpenPypeMongoConnection:
- """Singleton MongoDB connection.
- Keeps MongoDB connections by url.
- """
- mongo_clients = {}
- log = logging.getLogger("OpenPypeMongoConnection")
-
- @staticmethod
- def get_default_mongo_url():
- return os.environ["OPENPYPE_MONGO"]
-
- @classmethod
- def get_mongo_client(cls, mongo_url=None):
- if mongo_url is None:
- mongo_url = cls.get_default_mongo_url()
-
- connection = cls.mongo_clients.get(mongo_url)
- if connection:
- # Naive validation of existing connection
- try:
- connection.server_info()
- with connection.start_session():
- pass
- except Exception:
- connection = None
-
- if not connection:
- cls.log.debug("Creating mongo connection to {}".format(mongo_url))
- connection = cls.create_connection(mongo_url)
- cls.mongo_clients[mongo_url] = connection
-
- return connection
-
- @classmethod
- def create_connection(cls, mongo_url, timeout=None, retry_attempts=None):
- parsed = urlparse(mongo_url)
- # Force validation of scheme
- if parsed.scheme not in ["mongodb", "mongodb+srv"]:
- raise pymongo.errors.InvalidURI((
- "Invalid URI scheme:"
- " URI must begin with 'mongodb://' or 'mongodb+srv://'"
- ))
-
- if timeout is None:
- timeout = int(os.environ.get("AVALON_TIMEOUT") or 1000)
-
- kwargs = {
- "serverSelectionTimeoutMS": timeout
- }
- if should_add_certificate_path_to_mongo_url(mongo_url):
- kwargs["ssl_ca_certs"] = certifi.where()
-
- mongo_client = pymongo.MongoClient(mongo_url, **kwargs)
-
- if retry_attempts is None:
- retry_attempts = 3
-
- elif not retry_attempts:
- retry_attempts = 1
-
- last_exc = None
- valid = False
- t1 = time.time()
- for attempt in range(1, retry_attempts + 1):
- try:
- mongo_client.server_info()
- with mongo_client.start_session():
- pass
- valid = True
- break
-
- except Exception as exc:
- last_exc = exc
- if attempt < retry_attempts:
- cls.log.warning(
- "Attempt {} failed. Retrying... ".format(attempt)
- )
- time.sleep(1)
-
- if not valid:
- raise last_exc
-
- cls.log.info("Connected to {}, delay {:.3f}s".format(
- mongo_url, time.time() - t1
- ))
- return mongo_client
+__all__ = (
+ "MongoEnvNotSet",
+ "OpenPypeMongoConnection",
+ "get_default_components",
+ "should_add_certificate_path_to_mongo_url",
+ "validate_mongo_connection",
+)
diff --git a/openpype/lib/openpype_version.py b/openpype/lib/openpype_version.py
new file mode 100644
index 00000000000..d547d347558
--- /dev/null
+++ b/openpype/lib/openpype_version.py
@@ -0,0 +1,226 @@
+"""Lib access to OpenPypeVersion from igniter.
+
+Access to logic from igniter is available only for OpenPype processes.
+Is meant to be able check OpenPype versions for studio. The logic is dependent
+on igniter's inner logic of versions.
+
+Keep in mind that all functions except 'get_installed_version' does not return
+OpenPype version located in build but versions available in remote versions
+repository or locally available.
+"""
+
+import os
+import sys
+
+import openpype.version
+
+from .python_module_tools import import_filepath
+
+
+# ----------------------------------------
+# Functions independent on OpenPypeVersion
+# ----------------------------------------
+def get_openpype_version():
+ """Version of pype that is currently used."""
+ return openpype.version.__version__
+
+
+def get_build_version():
+ """OpenPype version of build."""
+ # Return OpenPype version if is running from code
+ if not is_running_from_build():
+ return get_openpype_version()
+
+ # Import `version.py` from build directory
+ version_filepath = os.path.join(
+ os.environ["OPENPYPE_ROOT"],
+ "openpype",
+ "version.py"
+ )
+ if not os.path.exists(version_filepath):
+ return None
+
+ module = import_filepath(version_filepath, "openpype_build_version")
+ return getattr(module, "__version__", None)
+
+
+def is_running_from_build():
+ """Determine if current process is running from build or code.
+
+ Returns:
+ bool: True if running from build.
+ """
+ executable_path = os.environ["OPENPYPE_EXECUTABLE"]
+ executable_filename = os.path.basename(executable_path)
+ if "python" in executable_filename.lower():
+ return False
+ return True
+
+
+def is_running_staging():
+ """Currently used OpenPype is staging version.
+
+ Returns:
+ bool: True if openpype version containt 'staging'.
+ """
+ if "staging" in get_openpype_version():
+ return True
+ return False
+
+
+# ----------------------------------------
+# Functions dependent on OpenPypeVersion
+# - Make sense to call only in OpenPype process
+# ----------------------------------------
+def get_OpenPypeVersion():
+ """Access to OpenPypeVersion class stored in sys modules."""
+ return sys.modules.get("OpenPypeVersion")
+
+
+def op_version_control_available():
+ """Check if current process has access to OpenPypeVersion."""
+ if get_OpenPypeVersion() is None:
+ return False
+ return True
+
+
+def get_installed_version():
+ """Get OpenPype version inside build.
+
+ This version is not returned by any other functions here.
+ """
+ if op_version_control_available():
+ return get_OpenPypeVersion().get_installed_version()
+ return None
+
+
+def get_available_versions(*args, **kwargs):
+ """Get list of available versions."""
+ if op_version_control_available():
+ return get_OpenPypeVersion().get_available_versions(
+ *args, **kwargs
+ )
+ return None
+
+
+def openpype_path_is_set():
+ """OpenPype repository path is set in settings."""
+ if op_version_control_available():
+ return get_OpenPypeVersion().openpype_path_is_set()
+ return None
+
+
+def openpype_path_is_accessible():
+ """OpenPype version repository path can be accessed."""
+ if op_version_control_available():
+ return get_OpenPypeVersion().openpype_path_is_accessible()
+ return None
+
+
+def get_local_versions(*args, **kwargs):
+ """OpenPype versions available on this workstation."""
+ if op_version_control_available():
+ return get_OpenPypeVersion().get_local_versions(*args, **kwargs)
+ return None
+
+
+def get_remote_versions(*args, **kwargs):
+ """OpenPype versions in repository path."""
+ if op_version_control_available():
+ return get_OpenPypeVersion().get_remote_versions(*args, **kwargs)
+ return None
+
+
+def get_latest_version(staging=None, local=None, remote=None):
+ """Get latest version from repository path."""
+ if staging is None:
+ staging = is_running_staging()
+ if op_version_control_available():
+ return get_OpenPypeVersion().get_latest_version(
+ staging=staging,
+ local=local,
+ remote=remote
+ )
+ return None
+
+
+def get_expected_studio_version(staging=None):
+ """Expected production or staging version in studio."""
+ if staging is None:
+ staging = is_running_staging()
+ if op_version_control_available():
+ return get_OpenPypeVersion().get_expected_studio_version(staging)
+ return None
+
+
+def get_expected_version(staging=None):
+ expected_version = get_expected_studio_version(staging)
+ if expected_version is None:
+ # Look for latest if expected version is not set in settings
+ expected_version = get_latest_version(
+ staging=staging,
+ remote=True
+ )
+ return expected_version
+
+
+def is_current_version_studio_latest():
+ """Is currently running OpenPype version which is defined by studio.
+
+ It is not recommended to ask in each process as there may be situations
+ when older OpenPype should be used. For example on farm. But it does make
+ sense in processes that can run for a long time.
+
+ Returns:
+ None: Can't determine. e.g. when running from code or the build is
+ too old.
+ bool: True when is using studio
+ """
+ output = None
+ # Skip if is not running from build or build does not support version
+ # control or path to folder with zip files is not accessible
+ if (
+ not is_running_from_build()
+ or not op_version_control_available()
+ or not openpype_path_is_accessible()
+ ):
+ return output
+
+ # Get OpenPypeVersion class
+ OpenPypeVersion = get_OpenPypeVersion()
+ # Convert current version to OpenPypeVersion object
+ current_version = OpenPypeVersion(version=get_openpype_version())
+
+ # Get expected version (from settings)
+ expected_version = get_expected_version()
+ # Check if current version is expected version
+ return current_version == expected_version
+
+
+def is_current_version_higher_than_expected():
+ """Is current OpenPype version higher than version defined by studio.
+
+ Returns:
+ None: Can't determine. e.g. when running from code or the build is
+ too old.
+ bool: True when is higher than studio version.
+ """
+ output = None
+ # Skip if is not running from build or build does not support version
+ # control or path to folder with zip files is not accessible
+ if (
+ not is_running_from_build()
+ or not op_version_control_available()
+ or not openpype_path_is_accessible()
+ ):
+ return output
+
+ # Get OpenPypeVersion class
+ OpenPypeVersion = get_OpenPypeVersion()
+ # Convert current version to OpenPypeVersion object
+ current_version = OpenPypeVersion(version=get_openpype_version())
+
+ # Get expected version (from settings)
+ expected_version = get_expected_version()
+ # Check if current version is expected version
+ return current_version > expected_version
diff --git a/openpype/lib/path_templates.py b/openpype/lib/path_templates.py
new file mode 100644
index 00000000000..b160054e380
--- /dev/null
+++ b/openpype/lib/path_templates.py
@@ -0,0 +1,828 @@
+import os
+import re
+import copy
+import numbers
+import collections
+
+import six
+
+KEY_PATTERN = re.compile(r"(\{.*?[^{0]*\})")
+KEY_PADDING_PATTERN = re.compile(r"([^:]+)\S+[><]\S+")
+SUB_DICT_PATTERN = re.compile(r"([^\[\]]+)")
+OPTIONAL_PATTERN = re.compile(r"(<.*?[^{0]*>)[^0-9]*?")
+
+
+def merge_dict(main_dict, enhance_dict):
+ """Merges dictionaries by keys.
+
+ Function call itself if value on key is again dictionary.
+
+ Args:
+ main_dict (dict): First dict to merge second one into.
+ enhance_dict (dict): Second dict to be merged.
+
+ Returns:
+ dict: Merged result.
+
+ .. note:: does not overrides whole value on first found key
+ but only values differences from enhance_dict
+
+ """
+ for key, value in enhance_dict.items():
+ if key not in main_dict:
+ main_dict[key] = value
+ elif isinstance(value, dict) and isinstance(main_dict[key], dict):
+ main_dict[key] = merge_dict(main_dict[key], value)
+ else:
+ main_dict[key] = value
+ return main_dict
+
+
+class TemplateMissingKey(Exception):
+ """Exception for cases when key does not exist in template."""
+
+ msg = "Template key does not exist: `{}`."
+
+ def __init__(self, parents):
+ parent_join = "".join(["[\"{0}\"]".format(key) for key in parents])
+ super(TemplateMissingKey, self).__init__(
+ self.msg.format(parent_join)
+ )
+
+
+class TemplateUnsolved(Exception):
+ """Exception for unsolved template when strict is set to True."""
+
+ msg = "Template \"{0}\" is unsolved.{1}{2}"
+ invalid_types_msg = " Keys with invalid DataType: `{0}`."
+ missing_keys_msg = " Missing keys: \"{0}\"."
+
+ def __init__(self, template, missing_keys, invalid_types):
+ invalid_type_items = []
+ for _key, _type in invalid_types.items():
+ invalid_type_items.append(
+ "\"{0}\" {1}".format(_key, str(_type))
+ )
+
+ invalid_types_msg = ""
+ if invalid_type_items:
+ invalid_types_msg = self.invalid_types_msg.format(
+ ", ".join(invalid_type_items)
+ )
+
+ missing_keys_msg = ""
+ if missing_keys:
+ missing_keys_msg = self.missing_keys_msg.format(
+ ", ".join(missing_keys)
+ )
+ super(TemplateUnsolved, self).__init__(
+ self.msg.format(template, missing_keys_msg, invalid_types_msg)
+ )
+
+
+class StringTemplate(object):
+ """String that can be formatted."""
+ def __init__(self, template):
+ if not isinstance(template, six.string_types):
+ raise TypeError("<{}> argument must be a string, not {}.".format(
+ self.__class__.__name__, str(type(template))
+ ))
+
+ self._template = template
+ parts = []
+ last_end_idx = 0
+ for item in KEY_PATTERN.finditer(template):
+ start, end = item.span()
+ if start > last_end_idx:
+ parts.append(template[last_end_idx:start])
+ parts.append(FormattingPart(template[start:end]))
+ last_end_idx = end
+
+ if last_end_idx < len(template):
+ parts.append(template[last_end_idx:len(template)])
+
+ new_parts = []
+ for part in parts:
+ if not isinstance(part, six.string_types):
+ new_parts.append(part)
+ continue
+
+ substr = ""
+ for char in part:
+ if char not in ("<", ">"):
+ substr += char
+ else:
+ if substr:
+ new_parts.append(substr)
+ new_parts.append(char)
+ substr = ""
+ if substr:
+ new_parts.append(substr)
+
+ self._parts = self.find_optional_parts(new_parts)
+
+ def __str__(self):
+ return self.template
+
+ def __repr__(self):
+ return "<{}> {}".format(self.__class__.__name__, self.template)
+
+ def __contains__(self, other):
+ return other in self.template
+
+ def replace(self, *args, **kwargs):
+ self._template = self.template.replace(*args, **kwargs)
+ return self
+
+ @property
+ def template(self):
+ return self._template
+
+ def format(self, data):
+ """ Figure out with whole formatting.
+
+ Separate advanced keys (*Like '{project[name]}') from string which must
+ be formatted separatelly in case of missing or incomplete keys in data.
+
+ Args:
+ data (dict): Containing keys to be filled into template.
+
+ Returns:
+ TemplateResult: Filled or partially filled template containing all
+ data needed or missing for filling template.
+ """
+ result = TemplatePartResult()
+ for part in self._parts:
+ if isinstance(part, six.string_types):
+ result.add_output(part)
+ else:
+ part.format(data, result)
+
+ invalid_types = result.invalid_types
+ invalid_types.update(result.invalid_optional_types)
+ invalid_types = result.split_keys_to_subdicts(invalid_types)
+
+ missing_keys = result.missing_keys
+ missing_keys |= result.missing_optional_keys
+
+ solved = result.solved
+ used_values = result.get_clean_used_values()
+
+ return TemplateResult(
+ result.output,
+ self.template,
+ solved,
+ used_values,
+ missing_keys,
+ invalid_types
+ )
+
+ def format_strict(self, *args, **kwargs):
+ result = self.format(*args, **kwargs)
+ result.validate()
+ return result
+
+ @classmethod
+ def format_template(cls, template, data):
+ objected_template = cls(template)
+ return objected_template.format(data)
+
+ @classmethod
+ def format_strict_template(cls, template, data):
+ objected_template = cls(template)
+ return objected_template.format_strict(data)
+
+ @staticmethod
+ def find_optional_parts(parts):
+ new_parts = []
+ tmp_parts = {}
+ counted_symb = -1
+ for part in parts:
+ if part == "<":
+ counted_symb += 1
+ tmp_parts[counted_symb] = []
+
+ elif part == ">":
+ if counted_symb > -1:
+ parts = tmp_parts.pop(counted_symb)
+ counted_symb -= 1
+ # If part contains only single string keep value
+ # unchanged
+ if parts:
+ # Remove optional start char
+ parts.pop(0)
+
+ if not parts:
+ value = "<>"
+ elif (
+ len(parts) == 1
+ and isinstance(parts[0], six.string_types)
+ ):
+ value = "<{}>".format(parts[0])
+ else:
+ value = OptionalPart(parts)
+
+ if counted_symb < 0:
+ out_parts = new_parts
+ else:
+ out_parts = tmp_parts[counted_symb]
+ # Store value
+ out_parts.append(value)
+ continue
+
+ if counted_symb < 0:
+ new_parts.append(part)
+ else:
+ tmp_parts[counted_symb].append(part)
+
+ if tmp_parts:
+ for idx in sorted(tmp_parts.keys()):
+ new_parts.extend(tmp_parts[idx])
+ return new_parts
+
+
+class TemplatesDict(object):
+ def __init__(self, templates=None):
+ self._raw_templates = None
+ self._templates = None
+ self._objected_templates = None
+ self.set_templates(templates)
+
+ def set_templates(self, templates):
+ if templates is None:
+ self._raw_templates = None
+ self._templates = None
+ self._objected_templates = None
+ elif isinstance(templates, dict):
+ self._raw_templates = copy.deepcopy(templates)
+ self._templates = templates
+ self._objected_templates = self.create_ojected_templates(templates)
+ else:
+ raise TypeError("<{}> argument must be a dict, not {}.".format(
+ self.__class__.__name__, str(type(templates))
+ ))
+
+ def __getitem__(self, key):
+ return self.templates[key]
+
+ def get(self, key, *args, **kwargs):
+ return self.templates.get(key, *args, **kwargs)
+
+ @property
+ def raw_templates(self):
+ return self._raw_templates
+
+ @property
+ def templates(self):
+ return self._templates
+
+ @property
+ def objected_templates(self):
+ return self._objected_templates
+
+ @classmethod
+ def create_ojected_templates(cls, templates):
+ if not isinstance(templates, dict):
+ raise TypeError("Expected dict object, got {}".format(
+ str(type(templates))
+ ))
+
+ objected_templates = copy.deepcopy(templates)
+ inner_queue = collections.deque()
+ inner_queue.append(objected_templates)
+ while inner_queue:
+ item = inner_queue.popleft()
+ if not isinstance(item, dict):
+ continue
+ for key in tuple(item.keys()):
+ value = item[key]
+ if isinstance(value, six.string_types):
+ item[key] = StringTemplate(value)
+ elif isinstance(value, dict):
+ inner_queue.append(value)
+ return objected_templates
+
+ def _format_value(self, value, data):
+ if isinstance(value, StringTemplate):
+ return value.format(data)
+
+ if isinstance(value, dict):
+ return self._solve_dict(value, data)
+ return value
+
+ def _solve_dict(self, templates, data):
+ """ Solves templates with entered data.
+
+ Args:
+ templates (dict): All templates which will be formatted.
+ data (dict): Containing keys to be filled into template.
+
+ Returns:
+ dict: With `TemplateResult` in values containing filled or
+ partially filled templates.
+ """
+ output = collections.defaultdict(dict)
+ for key, value in templates.items():
+ output[key] = self._format_value(value, data)
+
+ return output
+
+ def format(self, in_data, only_keys=True, strict=True):
+ """ Solves templates based on entered data.
+
+ Args:
+ data (dict): Containing keys to be filled into template.
+ only_keys (bool, optional): Decides if environ will be used to
+ fill templates or only keys in data.
+
+ Returns:
+ TemplatesResultDict: Output `TemplateResult` have `strict`
+ attribute set to True so accessing unfilled keys in templates
+ will raise exceptions with explaned error.
+ """
+ # Create a copy of inserted data
+ data = copy.deepcopy(in_data)
+
+ # Add environment variable to data
+ if only_keys is False:
+ for key, val in os.environ.items():
+ env_key = "$" + key
+ if env_key not in data:
+ data[env_key] = val
+
+ solved = self._solve_dict(self.objected_templates, data)
+
+ output = TemplatesResultDict(solved)
+ output.strict = strict
+ return output
+
+
+class TemplateResult(str):
+ """Result of template format with most of information in.
+
+ Args:
+ used_values (dict): Dictionary of template filling data with
+ only used keys.
+ solved (bool): For check if all required keys were filled.
+ template (str): Original template.
+ missing_keys (list): Missing keys that were not in the data. Include
+ missing optional keys.
+ invalid_types (dict): When key was found in data, but value had not
+ allowed DataType. Allowed data types are `numbers`,
+ `str`(`basestring`) and `dict`. Dictionary may cause invalid type
+ when value of key in data is dictionary but template expect string
+ of number.
+ """
+
+ used_values = None
+ solved = None
+ template = None
+ missing_keys = None
+ invalid_types = None
+
+ def __new__(
+ cls, filled_template, template, solved,
+ used_values, missing_keys, invalid_types
+ ):
+ new_obj = super(TemplateResult, cls).__new__(cls, filled_template)
+ new_obj.used_values = used_values
+ new_obj.solved = solved
+ new_obj.template = template
+ new_obj.missing_keys = list(set(missing_keys))
+ new_obj.invalid_types = invalid_types
+ return new_obj
+
+ def __copy__(self, *args, **kwargs):
+ return self.copy()
+
+ def __deepcopy__(self, *args, **kwargs):
+ return self.copy()
+
+ def validate(self):
+ if not self.solved:
+ raise TemplateUnsolved(
+ self.template,
+ self.missing_keys,
+ self.invalid_types
+ )
+
+ def copy(self):
+ cls = self.__class__
+ return cls(
+ str(self),
+ self.template,
+ self.solved,
+ self.used_values,
+ self.missing_keys,
+ self.invalid_types
+ )
+
+ def normalized(self):
+ """Convert to normalized path."""
+
+ cls = self.__class__
+ return cls(
+ os.path.normpath(self),
+ self.template,
+ self.solved,
+ self.used_values,
+ self.missing_keys,
+ self.invalid_types
+ )
+
+
+class TemplatesResultDict(dict):
+ """Holds and wrap TemplateResults for easy bug report."""
+
+ def __init__(self, in_data, key=None, parent=None, strict=None):
+ super(TemplatesResultDict, self).__init__()
+ for _key, _value in in_data.items():
+ if isinstance(_value, dict):
+ _value = self.__class__(_value, _key, self)
+ self[_key] = _value
+
+ self.key = key
+ self.parent = parent
+ self.strict = strict
+ if self.parent is None and strict is None:
+ self.strict = True
+
+ def __getitem__(self, key):
+ if key not in self.keys():
+ hier = self.hierarchy()
+ hier.append(key)
+ raise TemplateMissingKey(hier)
+
+ value = super(TemplatesResultDict, self).__getitem__(key)
+ if isinstance(value, self.__class__):
+ return value
+
+ # Raise exception when expected solved templates and it is not.
+ if self.raise_on_unsolved and hasattr(value, "validate"):
+ value.validate()
+ return value
+
+ @property
+ def raise_on_unsolved(self):
+ """To affect this change `strict` attribute."""
+ if self.strict is not None:
+ return self.strict
+ return self.parent.raise_on_unsolved
+
+ def hierarchy(self):
+ """Return dictionary keys one by one to root parent."""
+ if self.parent is None:
+ return []
+
+ hier_keys = []
+ par_hier = self.parent.hierarchy()
+ if par_hier:
+ hier_keys.extend(par_hier)
+ hier_keys.append(self.key)
+
+ return hier_keys
+
+ @property
+ def missing_keys(self):
+ """Return missing keys of all children templates."""
+ missing_keys = set()
+ for value in self.values():
+ missing_keys |= value.missing_keys
+ return missing_keys
+
+ @property
+ def invalid_types(self):
+ """Return invalid types of all children templates."""
+ invalid_types = {}
+ for value in self.values():
+ invalid_types = merge_dict(invalid_types, value.invalid_types)
+ return invalid_types
+
+ @property
+ def used_values(self):
+ """Return used values for all children templates."""
+ used_values = {}
+ for value in self.values():
+ used_values = merge_dict(used_values, value.used_values)
+ return used_values
+
+ def get_solved(self):
+ """Get only solved key from templates."""
+ result = {}
+ for key, value in self.items():
+ if isinstance(value, self.__class__):
+ value = value.get_solved()
+ if not value:
+ continue
+ result[key] = value
+
+ elif (
+ not hasattr(value, "solved") or
+ value.solved
+ ):
+ result[key] = value
+ return self.__class__(result, key=self.key, parent=self.parent)
+
+
+class TemplatePartResult:
+ """Result to store result of template parts."""
+ def __init__(self, optional=False):
+ # Missing keys or invalid value types of required keys
+ self._missing_keys = set()
+ self._invalid_types = {}
+ # Missing keys or invalid value types of optional keys
+ self._missing_optional_keys = set()
+ self._invalid_optional_types = {}
+
+ # Used values stored by key with origin type
+ # - key without any padding or key modifiers
+ # - value from filling data
+ # Example: {"version": 1}
+ self._used_values = {}
+ # Used values stored by key with all modifirs
+ # - value is already formatted string
+ # Example: {"version:0>3": "001"}
+ self._realy_used_values = {}
+ # Concatenated string output after formatting
+ self._output = ""
+ # Is this result from optional part
+ self._optional = True
+
+ def add_output(self, other):
+ if isinstance(other, six.string_types):
+ self._output += other
+
+ elif isinstance(other, TemplatePartResult):
+ self._output += other.output
+
+ self._missing_keys |= other.missing_keys
+ self._missing_optional_keys |= other.missing_optional_keys
+
+ self._invalid_types.update(other.invalid_types)
+ self._invalid_optional_types.update(other.invalid_optional_types)
+
+ if other.optional and not other.solved:
+ return
+ self._used_values.update(other.used_values)
+ self._realy_used_values.update(other.realy_used_values)
+
+ else:
+ raise TypeError("Cannot add data from \"{}\" to \"{}\"".format(
+ str(type(other)), self.__class__.__name__)
+ )
+
+ @property
+ def solved(self):
+ if self.optional:
+ if (
+ len(self.missing_optional_keys) > 0
+ or len(self.invalid_optional_types) > 0
+ ):
+ return False
+ return (
+ len(self.missing_keys) == 0
+ and len(self.invalid_types) == 0
+ )
+
+ @property
+ def optional(self):
+ return self._optional
+
+ @property
+ def output(self):
+ return self._output
+
+ @property
+ def missing_keys(self):
+ return self._missing_keys
+
+ @property
+ def missing_optional_keys(self):
+ return self._missing_optional_keys
+
+ @property
+ def invalid_types(self):
+ return self._invalid_types
+
+ @property
+ def invalid_optional_types(self):
+ return self._invalid_optional_types
+
+ @property
+ def realy_used_values(self):
+ return self._realy_used_values
+
+ @property
+ def used_values(self):
+ return self._used_values
+
+ @staticmethod
+ def split_keys_to_subdicts(values):
+ output = {}
+ for key, value in values.items():
+ key_padding = list(KEY_PADDING_PATTERN.findall(key))
+ if key_padding:
+ key = key_padding[0]
+ key_subdict = list(SUB_DICT_PATTERN.findall(key))
+ data = output
+ last_key = key_subdict.pop(-1)
+ for subkey in key_subdict:
+ if subkey not in data:
+ data[subkey] = {}
+ data = data[subkey]
+ data[last_key] = value
+ return output
+
+ def get_clean_used_values(self):
+ new_used_values = {}
+ for key, value in self.used_values.items():
+ if isinstance(value, FormatObject):
+ value = str(value)
+ new_used_values[key] = value
+
+ return self.split_keys_to_subdicts(new_used_values)
+
+ def add_realy_used_value(self, key, value):
+ self._realy_used_values[key] = value
+
+ def add_used_value(self, key, value):
+ self._used_values[key] = value
+
+ def add_missing_key(self, key):
+ if self._optional:
+ self._missing_optional_keys.add(key)
+ else:
+ self._missing_keys.add(key)
+
+ def add_invalid_type(self, key, value):
+ if self._optional:
+ self._invalid_optional_types[key] = type(value)
+ else:
+ self._invalid_types[key] = type(value)
+
+
+class FormatObject(object):
+ """Object that can be used for formatting.
+
+ This is base that is valid for to be used in 'StringTemplate' value.
+ """
+ def __init__(self):
+ self.value = ""
+
+ def __format__(self, *args, **kwargs):
+ return self.value.__format__(*args, **kwargs)
+
+ def __str__(self):
+ return str(self.value)
+
+ def __repr__(self):
+ return self.__str__()
+
+
+class FormattingPart:
+ """String with formatting template.
+
+ Containt only single key to format e.g. "{project[name]}".
+
+ Args:
+ template(str): String containing the formatting key.
+ """
+ def __init__(self, template):
+ self._template = template
+
+ @property
+ def template(self):
+ return self._template
+
+ def __repr__(self):
+ return "".format(self._template)
+
+ def __str__(self):
+ return self._template
+
+ @staticmethod
+ def validate_value_type(value):
+ """Check if value can be used for formatting of single key."""
+ if isinstance(value, (numbers.Number, FormatObject)):
+ return True
+
+ for inh_class in type(value).mro():
+ if inh_class in six.string_types:
+ return True
+ return False
+
+ def format(self, data, result):
+ """Format the formattings string.
+
+ Args:
+ data(dict): Data that should be used for formatting.
+ result(TemplatePartResult): Object where result is stored.
+ """
+ key = self.template[1:-1]
+ if key in result.realy_used_values:
+ result.add_output(result.realy_used_values[key])
+ return result
+
+ # check if key expects subdictionary keys (e.g. project[name])
+ existence_check = key
+ key_padding = list(KEY_PADDING_PATTERN.findall(existence_check))
+ if key_padding:
+ existence_check = key_padding[0]
+ key_subdict = list(SUB_DICT_PATTERN.findall(existence_check))
+
+ value = data
+ missing_key = False
+ invalid_type = False
+ used_keys = []
+ for sub_key in key_subdict:
+ if (
+ value is None
+ or (hasattr(value, "items") and sub_key not in value)
+ ):
+ missing_key = True
+ used_keys.append(sub_key)
+ break
+
+ if not hasattr(value, "items"):
+ invalid_type = True
+ break
+
+ used_keys.append(sub_key)
+ value = value.get(sub_key)
+
+ if missing_key or invalid_type:
+ if len(used_keys) == 0:
+ invalid_key = key_subdict[0]
+ else:
+ invalid_key = used_keys[0]
+ for idx, sub_key in enumerate(used_keys):
+ if idx == 0:
+ continue
+ invalid_key += "[{0}]".format(sub_key)
+
+ if missing_key:
+ result.add_missing_key(invalid_key)
+
+ elif invalid_type:
+ result.add_invalid_type(invalid_key, value)
+
+ result.add_output(self.template)
+ return result
+
+ if self.validate_value_type(value):
+ fill_data = {}
+ first_value = True
+ for used_key in reversed(used_keys):
+ if first_value:
+ first_value = False
+ fill_data[used_key] = value
+ else:
+ _fill_data = {used_key: fill_data}
+ fill_data = _fill_data
+
+ formatted_value = self.template.format(**fill_data)
+ result.add_realy_used_value(key, formatted_value)
+ result.add_used_value(existence_check, formatted_value)
+ result.add_output(formatted_value)
+ return result
+
+ result.add_invalid_type(key, value)
+ result.add_output(self.template)
+
+ return result
+
+
+class OptionalPart:
+ """Template part which contains optional formatting strings.
+
+ If this part can't be filled the result is empty string.
+
+ Args:
+ parts(list): Parts of template. Can contain 'str', 'OptionalPart' or
+ 'FormattingPart'.
+ """
+
+ def __init__(self, parts):
+ self._parts = parts
+
+ @property
+ def parts(self):
+ return self._parts
+
+ def __str__(self):
+ return "<{}>".format("".join([str(p) for p in self._parts]))
+
+ def __repr__(self):
+ return "".format("".join([str(p) for p in self._parts]))
+
+ def format(self, data, result):
+ new_result = TemplatePartResult(True)
+ for part in self._parts:
+ if isinstance(part, six.string_types):
+ new_result.add_output(part)
+ else:
+ part.format(data, new_result)
+
+ if new_result.solved:
+ result.add_output(new_result)
+ return result
diff --git a/openpype/lib/path_tools.py b/openpype/lib/path_tools.py
index 9bb0231ca74..0b6d0a3391a 100644
--- a/openpype/lib/path_tools.py
+++ b/openpype/lib/path_tools.py
@@ -1,17 +1,154 @@
-import json
-import logging
import os
import re
-import abc
-import six
-
+import logging
+import platform
+import functools
+import warnings
-from .anatomy import Anatomy
-from openpype.settings import get_project_settings
+import clique
log = logging.getLogger(__name__)
+class PathToolsDeprecatedWarning(DeprecationWarning):
+ pass
+
+
+def deprecated(new_destination):
+ """Mark functions as deprecated.
+
+ It will result in a warning being emitted when the function is used.
+ """
+
+ func = None
+ if callable(new_destination):
+ func = new_destination
+ new_destination = None
+
+ def _decorator(decorated_func):
+ if new_destination is None:
+ warning_message = (
+ " Please check content of deprecated function to figure out"
+ " possible replacement."
+ )
+ else:
+ warning_message = " Please replace your usage with '{}'.".format(
+ new_destination
+ )
+
+ @functools.wraps(decorated_func)
+ def wrapper(*args, **kwargs):
+ warnings.simplefilter("always", PathToolsDeprecatedWarning)
+ warnings.warn(
+ (
+ "Call to deprecated function '{}'"
+ "\nFunction was moved or removed.{}"
+ ).format(decorated_func.__name__, warning_message),
+ category=PathToolsDeprecatedWarning,
+ stacklevel=4
+ )
+ return decorated_func(*args, **kwargs)
+ return wrapper
+
+ if func is None:
+ return _decorator
+ return _decorator(func)
+
+
+def format_file_size(file_size, suffix=None):
+ """Returns formatted string with size in appropriate unit.
+
+ Args:
+ file_size (int): Size of file in bytes.
+ suffix (str): Suffix for formatted size. Default is 'B' (as bytes).
+
+ Returns:
+ str: Formatted size using proper unit and passed suffix (e.g. 7 MiB).
+ """
+
+ if suffix is None:
+ suffix = "B"
+
+ for unit in ["", "Ki", "Mi", "Gi", "Ti", "Pi", "Ei", "Zi"]:
+ if abs(file_size) < 1024.0:
+ return "%3.1f%s%s" % (file_size, unit, suffix)
+ file_size /= 1024.0
+ return "%.1f%s%s" % (file_size, "Yi", suffix)
+
+
+def create_hard_link(src_path, dst_path):
+ """Create hardlink of file.
+
+ Args:
+ src_path(str): Full path to a file which is used as source for
+ hardlink.
+ dst_path(str): Full path to a file where a link of source will be
+ added.
+ """
+ # Use `os.link` if is available
+ # - should be for all platforms with newer python versions
+ if hasattr(os, "link"):
+ os.link(src_path, dst_path)
+ return
+
+ # Windows implementation of hardlinks
+ # - used in Python 2
+ if platform.system().lower() == "windows":
+ import ctypes
+ from ctypes.wintypes import BOOL
+ CreateHardLink = ctypes.windll.kernel32.CreateHardLinkW
+ CreateHardLink.argtypes = [
+ ctypes.c_wchar_p, ctypes.c_wchar_p, ctypes.c_void_p
+ ]
+ CreateHardLink.restype = BOOL
+
+ res = CreateHardLink(dst_path, src_path, None)
+ if res == 0:
+ raise ctypes.WinError()
+ return
+ # Raises not implemented error if gets here
+ raise NotImplementedError(
+ "Implementation of hardlink for current environment is missing."
+ )
+
+
+def collect_frames(files):
+ """Returns dict of source path and its frame, if from sequence
+
+ Uses clique as most precise solution, used when anatomy template that
+ created files is not known.
+
+ Assumption is that frames are separated by '.', negative frames are not
+ allowed.
+
+ Args:
+ files(list) or (set with single value): list of source paths
+
+ Returns:
+ (dict): {'/asset/subset_v001.0001.png': '0001', ....}
+ """
+
+ patterns = [clique.PATTERNS["frames"]]
+ collections, remainder = clique.assemble(
+ files, minimum_items=1, patterns=patterns)
+
+ sources_and_frames = {}
+ if collections:
+ for collection in collections:
+ src_head = collection.head
+ src_tail = collection.tail
+
+ for index in collection.indexes:
+ src_frame = collection.format("{padding}") % index
+ src_file_name = "{}{}{}".format(
+ src_head, src_frame, src_tail)
+ sources_and_frames[src_file_name] = src_frame
+ else:
+ sources_and_frames[remainder.pop()] = None
+
+ return sources_and_frames
+
+
def _rreplace(s, a, b, n=1):
"""Replace a with b in string s from right side n times."""
return b.join(s.rsplit(a, n))
@@ -49,12 +186,6 @@ def version_up(filepath):
padding=padding)
new_label = label.replace(version, new_version, 1)
new_basename = _rreplace(basename, label, new_label)
-
- if not new_basename.endswith(new_label):
- index = (new_basename.find(new_label))
- index += len(new_label)
- new_basename = new_basename[:index]
-
new_filename = "{}{}".format(new_basename, ext)
new_filename = os.path.join(dirname, new_filename)
new_filename = os.path.normpath(new_filename)
@@ -63,8 +194,19 @@ def version_up(filepath):
raise RuntimeError("Created path is the same as current file,"
"this is a bug")
+ # We check for version clashes against the current file for any file
+ # that matches completely in name up to the {version} label found. Thus
+ # if source file was test_v001_test.txt we want to also check clashes
+ # against test_v002.txt but do want to preserve the part after the version
+ # label for our new filename
+ clash_basename = new_basename
+ if not clash_basename.endswith(new_label):
+ index = (clash_basename.find(new_label))
+ index += len(new_label)
+ clash_basename = clash_basename[:index]
+
for file in os.listdir(dirname):
- if file.endswith(ext) and file.startswith(new_basename):
+ if file.endswith(ext) and file.startswith(clash_basename):
log.info("Skipping existing version %s" % new_label)
return version_up(new_filename)
@@ -76,12 +218,12 @@ def get_version_from_path(file):
"""Find version number in file path string.
Args:
- file (string): file path
+ file (str): file path
Returns:
- v: version number in string ('001')
-
+ str: version number in string ('001')
"""
+
pattern = re.compile(r"[\._]v([0-9]+)", re.IGNORECASE)
try:
return pattern.findall(file)[-1]
@@ -97,16 +239,17 @@ def get_last_version_from_path(path_dir, filter):
"""Find last version of given directory content.
Args:
- path_dir (string): directory path
+ path_dir (str): directory path
filter (list): list of strings used as file name filter
Returns:
- string: file name with last version
+ str: file name with last version
Example:
last_version_file = get_last_version_from_path(
"/project/shots/shot01/work", ["shot01", "compositing", "nk"])
"""
+
assert os.path.isdir(path_dir), "`path_dir` argument needs to be directory"
assert isinstance(filter, list) and (
len(filter) != 0), "`filter` argument needs to be list and not empty"
@@ -114,10 +257,10 @@ def get_last_version_from_path(path_dir, filter):
filtred_files = list()
# form regex for filtering
- patern = r".*".join(filter)
+ pattern = r".*".join(filter)
for file in os.listdir(path_dir):
- if not re.findall(patern, file):
+ if not re.findall(pattern, file):
continue
filtred_files.append(file)
@@ -128,228 +271,97 @@ def get_last_version_from_path(path_dir, filter):
return None
-def compute_paths(basic_paths_items, project_root):
- pattern_array = re.compile(r"\[.*\]")
- project_root_key = "__project_root__"
- output = []
- for path_items in basic_paths_items:
- clean_items = []
- for path_item in path_items:
- matches = re.findall(pattern_array, path_item)
- if len(matches) > 0:
- path_item = path_item.replace(matches[0], "")
- if path_item == project_root_key:
- path_item = project_root
- clean_items.append(path_item)
- output.append(os.path.normpath(os.path.sep.join(clean_items)))
- return output
+@deprecated("openpype.pipeline.project_folders.concatenate_splitted_paths")
+def concatenate_splitted_paths(split_paths, anatomy):
+ """
+ Deprecated:
+ Function will be removed after release version 3.16.*
+ """
+
+ from openpype.pipeline.project_folders import concatenate_splitted_paths
+
+ return concatenate_splitted_paths(split_paths, anatomy)
+
+
+@deprecated
+def get_format_data(anatomy):
+ """
+ Deprecated:
+ Function will be removed after release version 3.16.*
+ """
+
+ from openpype.pipeline.template_data import get_project_template_data
+
+ data = get_project_template_data(project_name=anatomy.project_name)
+ data["root"] = anatomy.roots
+ return data
+@deprecated("openpype.pipeline.project_folders.fill_paths")
+def fill_paths(path_list, anatomy):
+ """
+ Deprecated:
+ Function will be removed after release version 3.16.*
+ """
+
+ from openpype.pipeline.project_folders import fill_paths
+
+ return fill_paths(path_list, anatomy)
+
+
+@deprecated("openpype.pipeline.project_folders.create_project_folders")
def create_project_folders(basic_paths, project_name):
- anatomy = Anatomy(project_name)
- roots_paths = []
- if isinstance(anatomy.roots, dict):
- for root in anatomy.roots.values():
- roots_paths.append(root.value)
- else:
- roots_paths.append(anatomy.roots.value)
-
- for root_path in roots_paths:
- project_root = os.path.join(root_path, project_name)
- full_paths = compute_paths(basic_paths, project_root)
- # Create folders
- for path in full_paths:
- full_path = path.format(project_root=project_root)
- if os.path.exists(full_path):
- log.debug(
- "Folder already exists: {}".format(full_path)
- )
- else:
- log.debug("Creating folder: {}".format(full_path))
- os.makedirs(full_path)
-
-
-def _list_path_items(folder_structure):
- output = []
- for key, value in folder_structure.items():
- if not value:
- output.append(key)
- else:
- paths = _list_path_items(value)
- for path in paths:
- if not isinstance(path, (list, tuple)):
- path = [path]
+ """
+ Deprecated:
+ Function will be removed after release version 3.16.*
+ """
- item = [key]
- item.extend(path)
- output.append(item)
+ from openpype.pipeline.project_folders import create_project_folders
- return output
+ return create_project_folders(project_name, basic_paths)
+@deprecated("openpype.pipeline.project_folders.get_project_basic_paths")
def get_project_basic_paths(project_name):
- project_settings = get_project_settings(project_name)
- folder_structure = (
- project_settings["global"]["project_folder_structure"]
- )
- if not folder_structure:
- return []
+ """
+ Deprecated:
+ Function will be removed after release version 3.16.*
+ """
- if isinstance(folder_structure, str):
- folder_structure = json.loads(folder_structure)
- return _list_path_items(folder_structure)
+ from openpype.pipeline.project_folders import get_project_basic_paths
+ return get_project_basic_paths(project_name)
-@six.add_metaclass(abc.ABCMeta)
-class HostDirmap:
- """
- Abstract class for running dirmap on a workfile in a host.
- Dirmap is used to translate paths inside of host workfile from one
- OS to another. (Eg. arstist created workfile on Win, different artists
- opens same file on Linux.)
+@deprecated("openpype.pipeline.workfile.create_workdir_extra_folders")
+def create_workdir_extra_folders(
+ workdir, host_name, task_type, task_name, project_name,
+ project_settings=None
+):
+ """Create extra folders in work directory based on context.
- Expects methods to be implemented inside of host:
- on_dirmap_enabled: run host code for enabling dirmap
- do_dirmap: run host code to do actual remapping
+ Args:
+ workdir (str): Path to workdir where workfiles is stored.
+ host_name (str): Name of host implementation.
+ task_type (str): Type of task for which extra folders should be
+ created.
+ task_name (str): Name of task for which extra folders should be
+ created.
+ project_name (str): Name of project on which task is.
+ project_settings (dict): Prepared project settings. Are loaded if not
+ passed.
+
+ Deprecated:
+ Function will be removed after release version 3.16.*
"""
- def __init__(self, host_name, project_settings, sync_module=None):
- self.host_name = host_name
- self.project_settings = project_settings
- self.sync_module = sync_module # to limit reinit of Modules
-
- self._mapping = None # cache mapping
-
- @abc.abstractmethod
- def on_enable_dirmap(self):
- """
- Run host dependent operation for enabling dirmap if necessary.
- """
-
- @abc.abstractmethod
- def dirmap_routine(self, source_path, destination_path):
- """
- Run host dependent remapping from source_path to destination_path
- """
-
- def process_dirmap(self):
- # type: (dict) -> None
- """Go through all paths in Settings and set them using `dirmap`.
-
- If artists has Site Sync enabled, take dirmap mapping directly from
- Local Settings when artist is syncing workfile locally.
-
- Args:
- project_settings (dict): Settings for current project.
-
- """
- if not self._mapping:
- self._mapping = self.get_mappings(self.project_settings)
- if not self._mapping:
- return
-
- log.info("Processing directory mapping ...")
- self.on_enable_dirmap()
- log.info("mapping:: {}".format(self._mapping))
-
- for k, sp in enumerate(self._mapping["source-path"]):
- try:
- print("{} -> {}".format(sp,
- self._mapping["destination-path"][k]))
- self.dirmap_routine(sp,
- self._mapping["destination-path"][k])
- except IndexError:
- # missing corresponding destination path
- log.error(("invalid dirmap mapping, missing corresponding"
- " destination directory."))
- break
- except RuntimeError:
- log.error("invalid path {} -> {}, mapping not registered".format( # noqa: E501
- sp, self._mapping["destination-path"][k]
- ))
- continue
-
- def get_mappings(self, project_settings):
- """Get translation from source-path to destination-path.
-
- It checks if Site Sync is enabled and user chose to use local
- site, in that case configuration in Local Settings takes precedence
- """
- local_mapping = self._get_local_sync_dirmap(project_settings)
- dirmap_label = "{}-dirmap".format(self.host_name)
- if not self.project_settings[self.host_name].get(dirmap_label) and \
- not local_mapping:
- return []
- mapping = local_mapping or \
- self.project_settings[self.host_name][dirmap_label]["paths"] or {}
- enbled = self.project_settings[self.host_name][dirmap_label]["enabled"]
- mapping_enabled = enbled or bool(local_mapping)
-
- if not mapping or not mapping_enabled or \
- not mapping.get("destination-path") or \
- not mapping.get("source-path"):
- return []
- return mapping
-
- def _get_local_sync_dirmap(self, project_settings):
- """
- Returns dirmap if synch to local project is enabled.
-
- Only valid mapping is from roots of remote site to local site set
- in Local Settings.
-
- Args:
- project_settings (dict)
- Returns:
- dict : { "source-path": [XXX], "destination-path": [YYYY]}
- """
- import json
- mapping = {}
-
- if not project_settings["global"]["sync_server"]["enabled"]:
- return mapping
-
- from openpype.settings.lib import get_site_local_overrides
-
- if not self.sync_module:
- from openpype.modules import ModulesManager
- manager = ModulesManager()
- self.sync_module = manager.modules_by_name["sync_server"]
-
- project_name = os.getenv("AVALON_PROJECT")
-
- active_site = self.sync_module.get_local_normalized_site(
- self.sync_module.get_active_site(project_name))
- remote_site = self.sync_module.get_local_normalized_site(
- self.sync_module.get_remote_site(project_name))
- log.debug("active {} - remote {}".format(active_site, remote_site))
-
- if active_site == "local" \
- and project_name in self.sync_module.get_enabled_projects()\
- and active_site != remote_site:
-
- sync_settings = self.sync_module.get_sync_project_setting(
- os.getenv("AVALON_PROJECT"), exclude_locals=False,
- cached=False)
-
- active_overrides = get_site_local_overrides(
- os.getenv("AVALON_PROJECT"), active_site)
- remote_overrides = get_site_local_overrides(
- os.getenv("AVALON_PROJECT"), remote_site)
-
- log.debug("local overrides".format(active_overrides))
- log.debug("remote overrides".format(remote_overrides))
- for root_name, active_site_dir in active_overrides.items():
- remote_site_dir = remote_overrides.get(root_name) or\
- sync_settings["sites"][remote_site]["root"][root_name]
- if os.path.isdir(active_site_dir):
- if not mapping.get("destination-path"):
- mapping["destination-path"] = []
- mapping["destination-path"].append(active_site_dir)
-
- if not mapping.get("source-path"):
- mapping["source-path"] = []
- mapping["source-path"].append(remote_site_dir)
-
- log.debug("local sync mapping:: {}".format(mapping))
- return mapping
+
+ from openpype.pipeline.project_folders import create_workdir_extra_folders
+
+ return create_workdir_extra_folders(
+ workdir,
+ host_name,
+ task_type,
+ task_name,
+ project_name,
+ project_settings
+ )
diff --git a/openpype/lib/plugin_tools.py b/openpype/lib/plugin_tools.py
index 7c66f9760de..1e157dfbfdc 100644
--- a/openpype/lib/plugin_tools.py
+++ b/openpype/lib/plugin_tools.py
@@ -1,29 +1,71 @@
# -*- coding: utf-8 -*-
"""Avalon/Pyblish plugin tools."""
import os
-import inspect
import logging
import re
-import json
-from .profiles_filtering import filter_profiles
+import warnings
+import functools
+from openpype.client import get_asset_by_id
from openpype.settings import get_project_settings
-
log = logging.getLogger(__name__)
-# Subset name template used when plugin does not have defined any
-DEFAULT_SUBSET_TEMPLATE = "{family}{Variant}"
+
+class PluginToolsDeprecatedWarning(DeprecationWarning):
+ pass
+
+
+def deprecated(new_destination):
+ """Mark functions as deprecated.
+
+ It will result in a warning being emitted when the function is used.
+ """
+
+ func = None
+ if callable(new_destination):
+ func = new_destination
+ new_destination = None
+
+ def _decorator(decorated_func):
+ if new_destination is None:
+ warning_message = (
+ " Please check content of deprecated function to figure out"
+ " possible replacement."
+ )
+ else:
+ warning_message = " Please replace your usage with '{}'.".format(
+ new_destination
+ )
+
+ @functools.wraps(decorated_func)
+ def wrapper(*args, **kwargs):
+ warnings.simplefilter("always", PluginToolsDeprecatedWarning)
+ warnings.warn(
+ (
+ "Call to deprecated function '{}'"
+ "\nFunction was moved or removed.{}"
+ ).format(decorated_func.__name__, warning_message),
+ category=PluginToolsDeprecatedWarning,
+ stacklevel=4
+ )
+ return decorated_func(*args, **kwargs)
+ return wrapper
+
+ if func is None:
+ return _decorator
+ return _decorator(func)
-class TaskNotSetError(KeyError):
- def __init__(self, msg=None):
- if not msg:
- msg = "Creator's subset name template requires task name."
- super(TaskNotSetError, self).__init__(msg)
+@deprecated("openpype.pipeline.create.TaskNotSetError")
+def TaskNotSetError(*args, **kwargs):
+ from openpype.pipeline.create import TaskNotSetError
+ return TaskNotSetError(*args, **kwargs)
+
+@deprecated("openpype.pipeline.create.get_subset_name")
def get_subset_name_with_asset_doc(
family,
variant,
@@ -62,61 +104,22 @@ def get_subset_name_with_asset_doc(
dbcon (AvalonMongoDB): Mongo connection to be able query asset document
if 'asset_doc' is not passed.
"""
- if not family:
- return ""
-
- if not host_name:
- host_name = os.environ["AVALON_APP"]
- # Use only last part of class family value split by dot (`.`)
- family = family.rsplit(".", 1)[-1]
+ from openpype.pipeline.create import get_subset_name
- if project_name is None:
- import avalon.api
-
- project_name = avalon.api.Session["AVALON_PROJECT"]
-
- asset_tasks = asset_doc.get("data", {}).get("tasks") or {}
- task_info = asset_tasks.get(task_name) or {}
- task_type = task_info.get("type")
-
- # Get settings
- tools_settings = get_project_settings(project_name)["global"]["tools"]
- profiles = tools_settings["creator"]["subset_name_profiles"]
- filtering_criteria = {
- "families": family,
- "hosts": host_name,
- "tasks": task_name,
- "task_types": task_type
- }
-
- matching_profile = filter_profiles(profiles, filtering_criteria)
- template = None
- if matching_profile:
- template = matching_profile["template"]
-
- # Make sure template is set (matching may have empty string)
- if not template:
- template = default_template or DEFAULT_SUBSET_TEMPLATE
-
- # Simple check of task name existence for template with {task} in
- # - missing task should be possible only in Standalone publisher
- if not task_name and "{task" in template.lower():
- raise TaskNotSetError()
-
- fill_pairs = {
- "variant": variant,
- "family": family,
- "task": task_name
- }
- if dynamic_data:
- # Dynamic data may override default values
- for key, value in dynamic_data.items():
- fill_pairs[key] = value
-
- return template.format(**prepare_template_data(fill_pairs))
+ return get_subset_name(
+ family,
+ variant,
+ task_name,
+ asset_doc,
+ project_name,
+ host_name,
+ default_template,
+ dynamic_data
+ )
+@deprecated
def get_subset_name(
family,
variant,
@@ -135,20 +138,15 @@ def get_subset_name(
This is legacy function should be replaced with
`get_subset_name_with_asset_doc` where asset document is expected.
"""
- if dbcon is None:
- from avalon.api import AvalonMongoDB
- dbcon = AvalonMongoDB()
- dbcon.Session["AVALON_PROJECT"] = project_name
+ from openpype.pipeline.create import get_subset_name
- dbcon.install()
+ if project_name is None:
+ project_name = dbcon.project_name
- asset_doc = dbcon.find_one(
- {"_id": asset_id},
- {"data.tasks": True}
- ) or {}
+ asset_doc = get_asset_by_id(project_name, asset_id, fields=["data.tasks"])
- return get_subset_name_with_asset_doc(
+ return get_subset_name(
family,
variant,
task_name,
@@ -164,7 +162,7 @@ def prepare_template_data(fill_pairs):
"""
Prepares formatted data for filling template.
- It produces mutliple variants of keys (key, Key, KEY) to control
+ It produces multiple variants of keys (key, Key, KEY) to control
format of filled template.
Args:
@@ -204,6 +202,7 @@ def prepare_template_data(fill_pairs):
return fill_data
+@deprecated("openpype.pipeline.publish.lib.filter_pyblish_plugins")
def filter_pyblish_plugins(plugins):
"""Filter pyblish plugins by presets.
@@ -214,56 +213,16 @@ def filter_pyblish_plugins(plugins):
plugins (dict): Dictionary of plugins produced by :mod:`pyblish-base`
`discover()` method.
+ Deprecated:
+ Function will be removed after release version 3.15.*
"""
- from pyblish import api
- host = api.current_host()
+ from openpype.pipeline.publish.lib import filter_pyblish_plugins
- presets = get_project_settings(os.environ['AVALON_PROJECT']) or {}
- # skip if there are no presets to process
- if not presets:
- return
-
- # iterate over plugins
- for plugin in plugins[:]:
-
- try:
- config_data = presets[host]["publish"][plugin.__name__]
- except KeyError:
- # host determined from path
- file = os.path.normpath(inspect.getsourcefile(plugin))
- file = os.path.normpath(file)
-
- split_path = file.split(os.path.sep)
- if len(split_path) < 4:
- log.warning(
- 'plugin path too short to extract host {}'.format(file)
- )
- continue
-
- host_from_file = split_path[-4]
- plugin_kind = split_path[-2]
-
- # TODO: change after all plugins are moved one level up
- if host_from_file == "openpype":
- host_from_file = "global"
-
- try:
- config_data = presets[host_from_file][plugin_kind][plugin.__name__] # noqa: E501
- except KeyError:
- continue
-
- for option, value in config_data.items():
- if option == "enabled" and value is False:
- log.info('removing plugin {}'.format(plugin.__name__))
- plugins.remove(plugin)
- else:
- log.info('setting {}:{} on plugin {}'.format(
- option, value, plugin.__name__))
-
- setattr(plugin, option, value)
+ filter_pyblish_plugins(plugins)
+@deprecated
def set_plugin_attributes_from_settings(
plugins, superclass, host_name=None, project_name=None
):
@@ -279,8 +238,14 @@ def set_plugin_attributes_from_settings(
Value from environment `AVALON_APP` is used if not entered.
project_name (str): Name of project for which settings will be loaded.
Value from environment `AVALON_PROJECT` is used if not entered.
+
+ Deprecated:
+ Function will be removed after release version 3.15.*
"""
+ # Function is not used anymore
+ from openpype.pipeline import LegacyCreator, LoaderPlugin
+
# determine host application to use for finding presets
if host_name is None:
host_name = os.environ.get("AVALON_APP")
@@ -288,12 +253,12 @@ def set_plugin_attributes_from_settings(
if project_name is None:
project_name = os.environ.get("AVALON_PROJECT")
- # map plugin superclass to preset json. Currenly suppoted is load and
- # create (avalon.api.Loader and avalon.api.Creator)
+ # map plugin superclass to preset json. Currently supported is load and
+ # create (LoaderPlugin and LegacyCreator)
plugin_type = None
- if superclass.__name__.split(".")[-1] in ("Loader", "SubsetLoader"):
+ if superclass is LoaderPlugin or issubclass(superclass, LoaderPlugin):
plugin_type = "load"
- elif superclass.__name__.split(".")[-1] == "Creator":
+ elif superclass is LegacyCreator or issubclass(superclass, LegacyCreator):
plugin_type = "create"
if not host_name or not project_name or plugin_type is None:
@@ -372,102 +337,3 @@ def source_hash(filepath, *args):
time = str(os.path.getmtime(filepath))
size = str(os.path.getsize(filepath))
return "|".join([file_name, time, size] + list(args)).replace(".", ",")
-
-
-def get_unique_layer_name(layers, name):
- """
- Gets all layer names and if 'name' is present in them, increases
- suffix by 1 (eg. creates unique layer name - for Loader)
- Args:
- layers (list): of strings, names only
- name (string): checked value
-
- Returns:
- (string): name_00X (without version)
- """
- names = {}
- for layer in layers:
- layer_name = re.sub(r'_\d{3}$', '', layer)
- if layer_name in names.keys():
- names[layer_name] = names[layer_name] + 1
- else:
- names[layer_name] = 1
- occurrences = names.get(name, 0)
-
- return "{}_{:0>3d}".format(name, occurrences + 1)
-
-
-def get_background_layers(file_url):
- """
- Pulls file name from background json file, enrich with folder url for
- AE to be able import files.
-
- Order is important, follows order in json.
-
- Args:
- file_url (str): abs url of background json
-
- Returns:
- (list): of abs paths to images
- """
- with open(file_url) as json_file:
- data = json.load(json_file)
-
- layers = list()
- bg_folder = os.path.dirname(file_url)
- for child in data['children']:
- if child.get("filename"):
- layers.append(os.path.join(bg_folder, child.get("filename")).
- replace("\\", "/"))
- else:
- for layer in child['children']:
- if layer.get("filename"):
- layers.append(os.path.join(bg_folder,
- layer.get("filename")).
- replace("\\", "/"))
- return layers
-
-
-def parse_json(path):
- """Parses json file at 'path' location
-
- Returns:
- (dict) or None if unparsable
- Raises:
- AsssertionError if 'path' doesn't exist
- """
- path = path.strip('\"')
- assert os.path.isfile(path), (
- "Path to json file doesn't exist. \"{}\"".format(path)
- )
- data = None
- with open(path, "r") as json_file:
- try:
- data = json.load(json_file)
- except Exception as exc:
- log.error(
- "Error loading json: "
- "{} - Exception: {}".format(path, exc)
- )
- return data
-
-
-def get_batch_asset_task_info(ctx):
- """Parses context data from webpublisher's batch metadata
-
- Returns:
- (tuple): asset, task_name (Optional), task_type
- """
- task_type = "default_task_type"
- task_name = None
- asset = None
-
- if ctx["type"] == "task":
- items = ctx["path"].split('/')
- asset = items[-2]
- task_name = ctx["name"]
- task_type = ctx["attributes"]["type"]
- else:
- asset = ctx["name"]
-
- return asset, task_name, task_type
diff --git a/openpype/lib/profiles_filtering.py b/openpype/lib/profiles_filtering.py
index 0bb901aff8d..370703a68bb 100644
--- a/openpype/lib/profiles_filtering.py
+++ b/openpype/lib/profiles_filtering.py
@@ -44,12 +44,6 @@ def _profile_exclusion(matching_profiles, logger):
Returns:
dict: Most matching profile.
"""
-
- logger.info(
- "Search for first most matching profile in match order:"
- " Host name -> Task name -> Family."
- )
-
if not matching_profiles:
return None
@@ -168,6 +162,15 @@ def filter_profiles(profiles_data, key_values, keys_order=None, logger=None):
_keys_order.append(key)
keys_order = tuple(_keys_order)
+ log_parts = " | ".join([
+ "{}: \"{}\"".format(*item)
+ for item in key_values.items()
+ ])
+
+ logger.info(
+ "Looking for matching profile for: {}".format(log_parts)
+ )
+
matching_profiles = None
highest_profile_points = -1
# Each profile get 1 point for each matching filter. Profile with most
@@ -205,11 +208,6 @@ def filter_profiles(profiles_data, key_values, keys_order=None, logger=None):
if profile_points == highest_profile_points:
matching_profiles.append((profile, profile_scores))
- log_parts = " | ".join([
- "{}: \"{}\"".format(*item)
- for item in key_values.items()
- ])
-
if not matching_profiles:
logger.info(
"None of profiles match your setup. {}".format(log_parts)
@@ -221,4 +219,9 @@ def filter_profiles(profiles_data, key_values, keys_order=None, logger=None):
"More than one profile match your setup. {}".format(log_parts)
)
- return _profile_exclusion(matching_profiles, logger)
+ profile = _profile_exclusion(matching_profiles, logger)
+ if profile:
+ logger.info(
+ "Profile selected: {}".format(profile)
+ )
+ return profile
diff --git a/openpype/lib/project_backpack.py b/openpype/lib/project_backpack.py
new file mode 100644
index 00000000000..ff2f1d4b883
--- /dev/null
+++ b/openpype/lib/project_backpack.py
@@ -0,0 +1,248 @@
+"""These lib functions are primarily for development purposes.
+
+WARNING: This is not meant for production data.
+
+Goal is to be able create package of current state of project with related
+documents from mongo and files from disk to zip file and then be able recreate
+the project based on the zip.
+
+This gives ability to create project where a changes and tests can be done.
+
+Keep in mind that to be able create a package of project has few requirements.
+Possible requirement should be listed in 'pack_project' function.
+"""
+import os
+import json
+import platform
+import tempfile
+import shutil
+import datetime
+
+import zipfile
+from bson.json_util import (
+ loads,
+ dumps,
+ CANONICAL_JSON_OPTIONS
+)
+from openpype.client import (
+ get_project,
+ get_whole_project,
+)
+from openpype.pipeline import AvalonMongoDB
+
+DOCUMENTS_FILE_NAME = "database"
+METADATA_FILE_NAME = "metadata"
+PROJECT_FILES_DIR = "project_files"
+
+
+def add_timestamp(filepath):
+ """Add timestamp string to a file."""
+ base, ext = os.path.splitext(filepath)
+ timestamp = datetime.datetime.now().strftime("%y%m%d_%H%M%S")
+ new_base = "{}_{}".format(base, timestamp)
+ return new_base + ext
+
+
+def pack_project(project_name, destination_dir=None):
+ """Make a package of a project with mongo documents and files.
+
+ This function has few restrictions:
+ - project must have only one root
+ - project must have all templates starting with
+ "{root[...]}/{project[name]}"
+
+ Args:
+ project_name(str): Project that should be packaged.
+ destination_dir(str): Optional path where zip will be stored. Project's
+ root is used if not passed.
+ """
+ print("Creating package of project \"{}\"".format(project_name))
+ # Validate existence of project
+ project_doc = get_project(project_name)
+ if not project_doc:
+ raise ValueError("Project \"{}\" was not found in database".format(
+ project_name
+ ))
+
+ roots = project_doc["config"]["roots"]
+ # Determine root directory of project
+ source_root = None
+ source_root_name = None
+ for root_name, root_value in roots.items():
+ if source_root is not None:
+ raise ValueError(
+ "Packaging is supported only for single root projects"
+ )
+ source_root = root_value
+ source_root_name = root_name
+
+ root_path = source_root[platform.system().lower()]
+ print("Using root \"{}\" with path \"{}\"".format(
+ source_root_name, root_path
+ ))
+
+ project_source_path = os.path.join(root_path, project_name)
+ if not os.path.exists(project_source_path):
+ raise ValueError("Didn't find source of project files")
+
+ # Determine zip filepath where data will be stored
+ if not destination_dir:
+ destination_dir = root_path
+
+ destination_dir = os.path.normpath(destination_dir)
+ if not os.path.exists(destination_dir):
+ os.makedirs(destination_dir)
+
+ zip_path = os.path.join(destination_dir, project_name + ".zip")
+
+ print("Project will be packaged into \"{}\"".format(zip_path))
+ # Rename already existing zip
+ if os.path.exists(zip_path):
+ dst_filepath = add_timestamp(zip_path)
+ os.rename(zip_path, dst_filepath)
+
+ # We can add more data
+ metadata = {
+ "project_name": project_name,
+ "root": source_root,
+ "version": 1
+ }
+ # Create temp json file where metadata are stored
+ with tempfile.NamedTemporaryFile("w", suffix=".json", delete=False) as s:
+ temp_metadata_json = s.name
+
+ with open(temp_metadata_json, "w") as stream:
+ json.dump(metadata, stream)
+
+ # Create temp json file where database documents are stored
+ with tempfile.NamedTemporaryFile("w", suffix=".json", delete=False) as s:
+ temp_docs_json = s.name
+
+ # Query all project documents and store them to temp json
+ docs = list(get_whole_project(project_name))
+ data = dumps(
+ docs, json_options=CANONICAL_JSON_OPTIONS
+ )
+ with open(temp_docs_json, "w") as stream:
+ stream.write(data)
+
+ print("Packing files into zip")
+ # Write all to zip file
+ with zipfile.ZipFile(zip_path, "w", zipfile.ZIP_DEFLATED) as zip_stream:
+ # Add metadata file
+ zip_stream.write(temp_metadata_json, METADATA_FILE_NAME + ".json")
+ # Add database documents
+ zip_stream.write(temp_docs_json, DOCUMENTS_FILE_NAME + ".json")
+ # Add project files to zip
+ for root, _, filenames in os.walk(project_source_path):
+ for filename in filenames:
+ filepath = os.path.join(root, filename)
+ # TODO add one more folder
+ archive_name = os.path.join(
+ PROJECT_FILES_DIR,
+ os.path.relpath(filepath, root_path)
+ )
+ zip_stream.write(filepath, archive_name)
+
+ print("Cleaning up")
+ # Cleanup
+ os.remove(temp_docs_json)
+ os.remove(temp_metadata_json)
+
+ print("*** Packing finished ***")
+
+
+def unpack_project(path_to_zip, new_root=None):
+ """Unpack project zip file to recreate project.
+
+ Args:
+ path_to_zip(str): Path to zip which was created using 'pack_project'
+ function.
+ new_root(str): Optional way how to set different root path for unpacked
+ project.
+ """
+ print("Unpacking project from zip {}".format(path_to_zip))
+ if not os.path.exists(path_to_zip):
+ print("Zip file does not exists: {}".format(path_to_zip))
+ return
+
+ tmp_dir = tempfile.mkdtemp(prefix="unpack_")
+ print("Zip is extracted to temp: {}".format(tmp_dir))
+ with zipfile.ZipFile(path_to_zip, "r") as zip_stream:
+ zip_stream.extractall(tmp_dir)
+
+ metadata_json_path = os.path.join(tmp_dir, METADATA_FILE_NAME + ".json")
+ with open(metadata_json_path, "r") as stream:
+ metadata = json.load(stream)
+
+ docs_json_path = os.path.join(tmp_dir, DOCUMENTS_FILE_NAME + ".json")
+ with open(docs_json_path, "r") as stream:
+ content = stream.readlines()
+ docs = loads("".join(content))
+
+ low_platform = platform.system().lower()
+ project_name = metadata["project_name"]
+ source_root = metadata["root"]
+ root_path = source_root[low_platform]
+
+ # Drop existing collection
+ dbcon = AvalonMongoDB()
+ database = dbcon.database
+ if project_name in database.list_collection_names():
+ database.drop_collection(project_name)
+ print("Removed existing project collection")
+
+ print("Creating project documents ({})".format(len(docs)))
+ # Create new collection with loaded docs
+ collection = database[project_name]
+ collection.insert_many(docs)
+
+ # Skip change of root if is the same as the one stored in metadata
+ if (
+ new_root
+ and (os.path.normpath(new_root) == os.path.normpath(root_path))
+ ):
+ new_root = None
+
+ if new_root:
+ print("Using different root path {}".format(new_root))
+ root_path = new_root
+
+ project_doc = get_project(project_name)
+ roots = project_doc["config"]["roots"]
+ key = tuple(roots.keys())[0]
+ update_key = "config.roots.{}.{}".format(key, low_platform)
+ collection.update_one(
+ {"_id": project_doc["_id"]},
+ {"$set": {
+ update_key: new_root
+ }}
+ )
+
+ # Make sure root path exists
+ if not os.path.exists(root_path):
+ os.makedirs(root_path)
+
+ src_project_files_dir = os.path.join(
+ tmp_dir, PROJECT_FILES_DIR, project_name
+ )
+ dst_project_files_dir = os.path.normpath(
+ os.path.join(root_path, project_name)
+ )
+ if os.path.exists(dst_project_files_dir):
+ new_path = add_timestamp(dst_project_files_dir)
+ print("Project folder already exists. Renamed \"{}\" -> \"{}\"".format(
+ dst_project_files_dir, new_path
+ ))
+ os.rename(dst_project_files_dir, new_path)
+
+ print("Moving project files from temp \"{}\" -> \"{}\"".format(
+ src_project_files_dir, dst_project_files_dir
+ ))
+ shutil.move(src_project_files_dir, dst_project_files_dir)
+
+ # CLeanup
+ print("Cleaning up")
+ shutil.rmtree(tmp_dir)
+ dbcon.uninstall()
+ print("*** Unpack finished ***")
diff --git a/openpype/lib/pype_info.py b/openpype/lib/pype_info.py
index 33715e369d0..8370ecc88f5 100644
--- a/openpype/lib/pype_info.py
+++ b/openpype/lib/pype_info.py
@@ -5,79 +5,26 @@
import getpass
import socket
-import openpype.version
from openpype.settings.lib import get_local_settings
-from .execute import get_pype_execute_args
+from .execute import get_openpype_execute_args
from .local_settings import get_local_site_id
-from .python_module_tools import import_filepath
+from .openpype_version import (
+ is_running_from_build,
+ get_openpype_version,
+ get_build_version
+)
-def get_openpype_version():
- """Version of pype that is currently used."""
- return openpype.version.__version__
-
-
-def get_pype_version():
- """Backwards compatibility. Remove when 100% not used."""
- print((
- "Using deprecated function 'openpype.lib.pype_info.get_pype_version'"
- " replace with 'openpype.lib.pype_info.get_openpype_version'."
- ))
- return get_openpype_version()
-
-
-def get_build_version():
- """OpenPype version of build."""
- # Return OpenPype version if is running from code
- if not is_running_from_build():
- return get_openpype_version()
-
- # Import `version.py` from build directory
- version_filepath = os.path.join(
- os.environ["OPENPYPE_ROOT"],
- "openpype",
- "version.py"
- )
- if not os.path.exists(version_filepath):
- return None
-
- module = import_filepath(version_filepath, "openpype_build_version")
- return getattr(module, "__version__", None)
-
-
-def is_running_from_build():
- """Determine if current process is running from build or code.
-
- Returns:
- bool: True if running from build.
- """
- executable_path = os.environ["OPENPYPE_EXECUTABLE"]
- executable_filename = os.path.basename(executable_path)
- if "python" in executable_filename.lower():
- return False
- return True
-
-
-def is_running_staging():
- """Currently used OpenPype is staging version.
-
- Returns:
- bool: True if openpype version containt 'staging'.
- """
- if "staging" in get_openpype_version():
- return True
- return False
-
-
-def get_pype_info():
+def get_openpype_info():
"""Information about currently used Pype process."""
- executable_args = get_pype_execute_args()
+ executable_args = get_openpype_execute_args()
if is_running_from_build():
version_type = "build"
else:
version_type = "code"
return {
+ "build_verison": get_build_version(),
"version": get_openpype_version(),
"version_type": version_type,
"executable": executable_args[-1],
@@ -106,7 +53,7 @@ def get_workstation_info():
def get_all_current_info():
"""All information about current process in one dictionary."""
return {
- "pype": get_pype_info(),
+ "pype": get_openpype_info(),
"workstation": get_workstation_info(),
"env": os.environ.copy(),
"local_settings": get_local_settings()
diff --git a/openpype/lib/python_module_tools.py b/openpype/lib/python_module_tools.py
index 69da4cc6619..6fad3b547fa 100644
--- a/openpype/lib/python_module_tools.py
+++ b/openpype/lib/python_module_tools.py
@@ -5,8 +5,9 @@
import inspect
import logging
+import six
+
log = logging.getLogger(__name__)
-PY3 = sys.version_info[0] == 3
def import_filepath(filepath, module_name=None):
@@ -28,7 +29,7 @@ def import_filepath(filepath, module_name=None):
# Prepare module object where content of file will be parsed
module = types.ModuleType(module_name)
- if PY3:
+ if six.PY3:
# Use loader so module has full specs
module_loader = importlib.machinery.SourceFileLoader(
module_name, filepath
@@ -38,7 +39,7 @@ def import_filepath(filepath, module_name=None):
# Execute module code and store content to module
with open(filepath) as _stream:
# Execute content and store it to module object
- exec(_stream.read(), module.__dict__)
+ six.exec_(_stream.read(), module.__dict__)
module.__file__ = filepath
return module
@@ -49,32 +50,30 @@ def modules_from_path(folder_path):
Arguments:
path (str): Path to folder containing python scripts.
- return_crasher (bool): Crashed module paths with exception info
- will be returned too.
Returns:
- list, tuple: List of modules when `return_crashed` is False else tuple
- with list of modules at first place and tuple of path and exception
- info at second place.
+ tuple: First list contains successfully imported modules
+ and second list contains tuples of path and exception.
"""
crashed = []
modules = []
+ output = (modules, crashed)
# Just skip and return empty list if path is not set
if not folder_path:
- return modules
+ return output
# Do not allow relative imports
if folder_path.startswith("."):
log.warning((
"BUG: Relative paths are not allowed for security reasons. {}"
).format(folder_path))
- return modules
+ return output
folder_path = os.path.normpath(folder_path)
if not os.path.isdir(folder_path):
log.warning("Not a directory path: {}".format(folder_path))
- return modules
+ return output
for filename in os.listdir(folder_path):
# Ignore files which start with underscore
@@ -101,7 +100,7 @@ def modules_from_path(folder_path):
)
continue
- return modules, crashed
+ return output
def recursive_bases_from_class(klass):
@@ -131,20 +130,12 @@ def classes_from_module(superclass, module):
for name in dir(module):
# It could be anything at this point
obj = getattr(module, name)
- if not inspect.isclass(obj):
- continue
-
- # These are subclassed from nothing, not even `object`
- if not len(obj.__bases__) > 0:
+ if not inspect.isclass(obj) or obj is superclass:
continue
- # Use string comparison rather than `issubclass`
- # in order to support reloading of this module.
- bases = recursive_bases_from_class(obj)
- if not any(base.__name__ == superclass.__name__ for base in bases):
- continue
+ if issubclass(obj, superclass):
+ classes.append(obj)
- classes.append(obj)
return classes
@@ -230,7 +221,7 @@ def import_module_from_dirpath(dirpath, folder_name, dst_module_name=None):
dst_module_name(str): Parent module name under which can be loaded
module added.
"""
- if PY3:
+ if six.PY3:
module = _import_module_from_dirpath_py3(
dirpath, folder_name, dst_module_name
)
diff --git a/openpype/lib/remote_publish.py b/openpype/lib/remote_publish.py
deleted file mode 100644
index 8074b2d1127..00000000000
--- a/openpype/lib/remote_publish.py
+++ /dev/null
@@ -1,235 +0,0 @@
-import os
-from datetime import datetime
-import sys
-from bson.objectid import ObjectId
-import collections
-
-import pyblish.util
-import pyblish.api
-
-from openpype import uninstall
-from openpype.lib.mongo import OpenPypeMongoConnection
-from openpype.lib.plugin_tools import parse_json
-
-
-def headless_publish(log, close_plugin_name=None, is_test=False):
- """Runs publish in a opened host with a context and closes Python process.
-
- Host is being closed via ClosePS pyblish plugin which triggers 'exit'
- method in ConsoleTrayApp.
- """
- if not is_test:
- dbcon = get_webpublish_conn()
- _id = os.environ.get("BATCH_LOG_ID")
- if not _id:
- log.warning("Unable to store log records, "
- "batch will be unfinished!")
- return
-
- publish_and_log(dbcon, _id, log, close_plugin_name)
- else:
- publish(log, close_plugin_name)
-
-
-def get_webpublish_conn():
- """Get connection to OP 'webpublishes' collection."""
- mongo_client = OpenPypeMongoConnection.get_mongo_client()
- database_name = os.environ["OPENPYPE_DATABASE_NAME"]
- return mongo_client[database_name]["webpublishes"]
-
-
-def start_webpublish_log(dbcon, batch_id, user):
- """Start new log record for 'batch_id'
-
- Args:
- dbcon (OpenPypeMongoConnection)
- batch_id (str)
- user (str)
- Returns
- (ObjectId) from DB
- """
- return dbcon.insert_one({
- "batch_id": batch_id,
- "start_date": datetime.now(),
- "user": user,
- "status": "in_progress",
- "progress": 0.0
- }).inserted_id
-
-
-def publish(log, close_plugin_name=None):
- """Loops through all plugins, logs to console. Used for tests.
-
- Args:
- log (OpenPypeLogger)
- close_plugin_name (str): name of plugin with responsibility to
- close host app
- """
- # Error exit as soon as any error occurs.
- error_format = "Failed {plugin.__name__}: {error} -- {error.traceback}"
-
- close_plugin = _get_close_plugin(close_plugin_name, log)
-
- for result in pyblish.util.publish_iter():
- for record in result["records"]:
- log.info("{}: {}".format(
- result["plugin"].label, record.msg))
-
- if result["error"]:
- log.error(error_format.format(**result))
- uninstall()
- if close_plugin: # close host app explicitly after error
- context = pyblish.api.Context()
- close_plugin().process(context)
- sys.exit(1)
-
-
-def publish_and_log(dbcon, _id, log, close_plugin_name=None):
- """Loops through all plugins, logs ok and fails into OP DB.
-
- Args:
- dbcon (OpenPypeMongoConnection)
- _id (str)
- log (OpenPypeLogger)
- close_plugin_name (str): name of plugin with responsibility to
- close host app
- """
- # Error exit as soon as any error occurs.
- error_format = "Failed {plugin.__name__}: {error} -- {error.traceback}"
-
- close_plugin = _get_close_plugin(close_plugin_name, log)
-
- if isinstance(_id, str):
- _id = ObjectId(_id)
-
- log_lines = []
- for result in pyblish.util.publish_iter():
- for record in result["records"]:
- log_lines.append("{}: {}".format(
- result["plugin"].label, record.msg))
-
- if result["error"]:
- log.error(error_format.format(**result))
- uninstall()
- log_lines.append(error_format.format(**result))
- dbcon.update_one(
- {"_id": _id},
- {"$set":
- {
- "finish_date": datetime.now(),
- "status": "error",
- "log": os.linesep.join(log_lines)
-
- }}
- )
- if close_plugin: # close host app explicitly after error
- context = pyblish.api.Context()
- close_plugin().process(context)
- sys.exit(1)
- else:
- dbcon.update_one(
- {"_id": _id},
- {"$set":
- {
- "progress": max(result["progress"], 0.95),
- "log": os.linesep.join(log_lines)
- }}
- )
-
- # final update
- dbcon.update_one(
- {"_id": _id},
- {"$set":
- {
- "finish_date": datetime.now(),
- "status": "finished_ok",
- "progress": 1,
- "log": os.linesep.join(log_lines)
- }}
- )
-
-
-def fail_batch(_id, batches_in_progress, dbcon):
- """Set current batch as failed as there are some stuck batches."""
- running_batches = [str(batch["_id"])
- for batch in batches_in_progress
- if batch["_id"] != _id]
- msg = "There are still running batches {}\n". \
- format("\n".join(running_batches))
- msg += "Ask admin to check them and reprocess current batch"
- dbcon.update_one(
- {"_id": _id},
- {"$set":
- {
- "finish_date": datetime.now(),
- "status": "error",
- "log": msg
-
- }}
- )
- raise ValueError(msg)
-
-
-def find_variant_key(application_manager, host):
- """Searches for latest installed variant for 'host'
-
- Args:
- application_manager (ApplicationManager)
- host (str)
- Returns
- (string) (optional)
- Raises:
- (ValueError) if no variant found
- """
- app_group = application_manager.app_groups.get(host)
- if not app_group or not app_group.enabled:
- raise ValueError("No application {} configured".format(host))
-
- found_variant_key = None
- # finds most up-to-date variant if any installed
- sorted_variants = collections.OrderedDict(
- sorted(app_group.variants.items()))
- for variant_key, variant in sorted_variants.items():
- for executable in variant.executables:
- if executable.exists():
- found_variant_key = variant_key
-
- if not found_variant_key:
- raise ValueError("No executable for {} found".format(host))
-
- return found_variant_key
-
-
-def _get_close_plugin(close_plugin_name, log):
- if close_plugin_name:
- plugins = pyblish.api.discover()
- for plugin in plugins:
- if plugin.__name__ == close_plugin_name:
- return plugin
-
- log.warning("Close plugin not found, app might not close.")
-
-
-def get_task_data(batch_dir):
- """Return parsed data from first task manifest.json
-
- Used for `remotepublishfromapp` command where batch contains only
- single task with publishable workfile.
-
- Returns:
- (dict)
- Throws:
- (ValueError) if batch or task manifest not found or broken
- """
- batch_data = parse_json(os.path.join(batch_dir, "manifest.json"))
- if not batch_data:
- raise ValueError(
- "Cannot parse batch meta in {} folder".format(batch_dir))
- task_dir_name = batch_data["tasks"][0]
- task_data = parse_json(os.path.join(batch_dir, task_dir_name,
- "manifest.json"))
- if not task_data:
- raise ValueError(
- "Cannot parse batch meta in {} folder".format(task_data))
-
- return task_data
diff --git a/openpype/lib/splash.txt b/openpype/lib/splash.txt
deleted file mode 100644
index 833bcd4b9cf..00000000000
--- a/openpype/lib/splash.txt
+++ /dev/null
@@ -1,413 +0,0 @@
-
-
-
- *
-
-
-
-
-
-
- .*
-
-
-
-
-
- *
- .*
- *
-
-
-
- .
- *
- .*
- *
- .
-
- .
- *
- .*
- .*
- .*
- *
- .
- .
- *
- .*
- .*
- .*
- *
- .
- _.
- /**
- \ *
- \*
- *
- *
- .
- __.
- ---*
- \ \*
- \ *
- \*
- *
- .
- \___.
- /* *
- \ \ *
- \ \*
- \ *
- \*
- .
- |____.
- /* *
- \|\ *
- \ \ *
- \ \ *
- \ \*
- \/.
- _/_____.
- /* *
- / \ *
- \ \ *
- \ \ *
- \ \__*
- \/__.
- __________.
- --*-- ___*
- \ \ \/_*
- \ \ __*
- \ \ \_*
- \ \____\*
- \/____/.
- \____________ .
- /* ___ \*
- \ \ \/_\ *
- \ \ _____*
- \ \ \___/*
- \ \____\ *
- \/____/ .
- |___________ .
- /* ___ \ *
- \|\ \/_\ \ *
- \ \ _____/ *
- \ \ \___/ *
- \ \____\ / *
- \/____/ \.
- _/__________ .
- /* ___ \ *
- / \ \/_\ \ *
- \ \ _____/ *
- \ \ \___/ ---*
- \ \____\ / \__*
- \/____/ \/__.
- ____________ .
- --*-- ___ \ *
- \ \ \/_\ \ *
- \ \ _____/ *
- \ \ \___/ ---- *
- \ \____\ / \____\*
- \/____/ \/____/.
- ____________
- /\ ___ \ .
- \ \ \/_\ \ *
- \ \ _____/ *
- \ \ \___/ ---- *
- \ \____\ / \____\ .
- \/____/ \/____/
- ____________
- /\ ___ \
- \ \ \/_\ \ .
- \ \ _____/ *
- \ \ \___/ ---- *
- \ \____\ / \____\ .
- \/____/ \/____/
- ____________
- /\ ___ \
- \ \ \/_\ \
- \ \ _____/ .
- \ \ \___/ ---- *
- \ \____\ / \____\ .
- \/____/ \/____/
- ____________
- /\ ___ \
- \ \ \/_\ \
- \ \ _____/
- \ \ \___/ ---- *
- \ \____\ / \____\
- \/____/ \/____/
- ____________
- /\ ___ \
- \ \ \/_\ \
- \ \ _____/
- \ \ \___/ ---- .
- \ \____\ / \____\
- \/____/ \/____/
- ____________
- /\ ___ \
- \ \ \/_\ \
- \ \ _____/ _
- \ \ \___/ ----
- \ \____\ / \____\
- \/____/ \/____/
- ____________
- /\ ___ \
- \ \ \/_\ \
- \ \ _____/ ___
- \ \ \___/ ----
- \ \____\ / \____\
- \/____/ \/____/
- ____________
- /\ ___ \
- \ \ \/_\ \
- \ \ _____/ ___
- \ \ \___/ ---- \
- \ \____\ / \____\
- \/____/ \/____/
- ____________
- /\ ___ \
- \ \ \/_\ \
- \ \ _____/ ___
- \ \ \___/ ---- \
- \ \____\ / \____\ \
- \/____/ \/____/
- ____________
- /\ ___ \
- \ \ \/_\ \
- \ \ _____/ ___
- \ \ \___/ ---- \
- \ \____\ / \____\ __\
- \/____/ \/____/
- ____________
- /\ ___ \
- \ \ \/_\ \
- \ \ _____/ ___
- \ \ \___/ ---- \
- \ \____\ / \____\ \__\
- \/____/ \/____/
- ____________
- /\ ___ \
- \ \ \/_\ \
- \ \ _____/ ___
- \ \ \___/ ---- \ \
- \ \____\ / \____\ \__\
- \/____/ \/____/
- ____________
- /\ ___ \
- \ \ \/_\ \
- \ \ _____/ ___
- \ \ \___/ ---- \ \
- \ \____\ / \____\ \__\
- \/____/ \/____/
- ____________
- /\ ___ \
- \ \ \/_\ \
- \ \ _____/ ___.
- \ \ \___/ ---- \ \\
- \ \____\ / \____\ \__\,
- \/____/ \/____/
- ____________
- /\ ___ \
- \ \ \/_\ \
- \ \ _____/ ___ .
- \ \ \___/ ---- \ \\
- \ \____\ / \____\ \__\\,
- \/____/ \/____/
- ____________
- /\ ___ \
- \ \ \/_\ \
- \ \ _____/ ___ _.
- \ \ \___/ ---- \ \\\
- \ \____\ / \____\ \__\\\
- \/____/ \/____/
- ____________
- /\ ___ \
- \ \ \/_\ \
- \ \ _____/ ___ __.
- \ \ \___/ ---- \ \\ \
- \ \____\ / \____\ \__\\_/.
- \/____/ \/____/
- ____________
- /\ ___ \
- \ \ \/_\ \
- \ \ _____/ ___ ___.
- \ \ \___/ ---- \ \\ \\
- \ \____\ / \____\ \__\\__\.
- \/____/ \/____/
- ____________
- /\ ___ \
- \ \ \/_\ \
- \ \ _____/ ___ ___ .
- \ \ \___/ ---- \ \\ \\
- \ \____\ / \____\ \__\\__\\.
- \/____/ \/____/
- ____________
- /\ ___ \
- \ \ \/_\ \
- \ \ _____/ ___ ___ _.
- \ \ \___/ ---- \ \\ \\\
- \ \____\ / \____\ \__\\__\\.
- \/____/ \/____/
- ____________
- /\ ___ \
- \ \ \/_\ \
- \ \ _____/ ___ ___ __.
- \ \ \___/ ---- \ \\ \\ \
- \ \____\ / \____\ \__\\__\\_.
- \/____/ \/____/
- ____________
- /\ ___ \
- \ \ \/_\ \
- \ \ _____/ ___ ___ __.
- \ \ \___/ ---- \ \\ \\ \
- \ \____\ / \____\ \__\\__\\__.
- \/____/ \/____/
- ____________
- /\ ___ \
- \ \ \/_\ \
- \ \ _____/ ___ ___ ___
- \ \ \___/ ---- \ \\ \\ \
- \ \____\ / \____\ \__\\__\\__\
- \/____/ \/____/ .
- ____________
- /\ ___ \
- \ \ \/_\ \
- \ \ _____/ ___ ___ ___
- \ \ \___/ ---- \ \\ \\ \
- \ \____\ / \____\ \__\\__\\__\
- \/____/ \/____/ *
- ____________
- /\ ___ \
- \ \ \/_\ \
- \ \ _____/ ___ ___ ___
- \ \ \___/ ---- \ \\ \\ \
- \ \____\ / \____\ \__\\__\\__\
- \/____/ \/____/ O*
- ____________
- /\ ___ \
- \ \ \/_\ \
- \ \ _____/ ___ ___ ___
- \ \ \___/ ---- \ \\ \\ \
- \ \____\ / \____\ \__\\__\\__\
- \/____/ \/____/ oO*
- ____________
- /\ ___ \
- \ \ \/_\ \
- \ \ _____/ ___ ___ ___
- \ \ \___/ ---- \ \\ \\ \
- \ \____\ / \____\ \__\\__\\__\
- \/____/ \/____/ .oO*
- ____________
- /\ ___ \
- \ \ \/_\ \
- \ \ _____/ ___ ___ ___
- \ \ \___/ ---- \ \\ \\ \
- \ \____\ / \____\ \__\\__\\__\
- \/____/ \/____/ ..oO*
- ____________
- /\ ___ \
- \ \ \/_\ \
- \ \ _____/ ___ ___ ___
- \ \ \___/ ---- \ \\ \\ \
- \ \____\ / \____\ \__\\__\\__\
- \/____/ \/____/ . .oO*
- ____________
- /\ ___ \
- \ \ \/_\ \
- \ \ _____/ ___ ___ ___
- \ \ \___/ ---- \ \\ \\ \
- \ \____\ / \____\ \__\\__\\__\
- \/____/ \/____/ . p.oO*
- ____________
- /\ ___ \
- \ \ \/_\ \
- \ \ _____/ ___ ___ ___
- \ \ \___/ ---- \ \\ \\ \
- \ \____\ / \____\ \__\\__\\__\
- \/____/ \/____/ . Py.oO*
- ____________
- /\ ___ \
- \ \ \/_\ \
- \ \ _____/ ___ ___ ___
- \ \ \___/ ---- \ \\ \\ \
- \ \____\ / \____\ \__\\__\\__\
- \/____/ \/____/ . PYp.oO*
- ____________
- /\ ___ \
- \ \ \/_\ \
- \ \ _____/ ___ ___ ___
- \ \ \___/ ---- \ \\ \\ \
- \ \____\ / \____\ \__\\__\\__\
- \/____/ \/____/ . PYPe.oO*
- ____________
- /\ ___ \
- \ \ \/_\ \
- \ \ _____/ ___ ___ ___
- \ \ \___/ ---- \ \\ \\ \
- \ \____\ / \____\ \__\\__\\__\
- \/____/ \/____/ . PYPE .oO*
- ____________
- /\ ___ \
- \ \ \/_\ \
- \ \ _____/ ___ ___ ___
- \ \ \___/ ---- \ \\ \\ \
- \ \____\ / \____\ \__\\__\\__\
- \/____/ \/____/ . PYPE c.oO*
- ____________
- /\ ___ \
- \ \ \/_\ \
- \ \ _____/ ___ ___ ___
- \ \ \___/ ---- \ \\ \\ \
- \ \____\ / \____\ \__\\__\\__\
- \/____/ \/____/ . PYPE C1.oO*
- ____________
- /\ ___ \
- \ \ \/_\ \
- \ \ _____/ ___ ___ ___
- \ \ \___/ ---- \ \\ \\ \
- \ \____\ / \____\ \__\\__\\__\
- \/____/ \/____/ . PYPE ClU.oO*
- ____________
- /\ ___ \
- \ \ \/_\ \
- \ \ _____/ ___ ___ ___
- \ \ \___/ ---- \ \\ \\ \
- \ \____\ / \____\ \__\\__\\__\
- \/____/ \/____/ . PYPE CluB.oO*
- ____________
- /\ ___ \
- \ \ \/_\ \
- \ \ _____/ ___ ___ ___
- \ \ \___/ ---- \ \\ \\ \
- \ \____\ / \____\ \__\\__\\__\
- \/____/ \/____/ . PYPE Club .oO*
- ____________
- /\ ___ \
- \ \ \/_\ \
- \ \ _____/ ___ ___ ___
- \ \ \___/ ---- \ \\ \\ \
- \ \____\ / \____\ \__\\__\\__\
- \/____/ \/____/ . PYPE Club . ..
- ____________
- /\ ___ \
- \ \ \/_\ \
- \ \ _____/ ___ ___ ___
- \ \ \___/ ---- \ \\ \\ \
- \ \____\ / \____\ \__\\__\\__\
- \/____/ \/____/ . PYPE Club . ..
- ____________
- /\ ___ \
- \ \ \/_\ \
- \ \ _____/ ___ ___ ___
- \ \ \___/ ---- \ \\ \\ \
- \ \____\ / \____\ \__\\__\\__\
- \/____/ \/____/ . PYPE Club . .
- ____________
- /\ ___ \
- \ \ \/_\ \
- \ \ _____/ ___ ___ ___
- \ \ \___/ ---- \ \\ \\ \
- \ \____\ / \____\ \__\\__\\__\
- \/____/ \/____/ . PYPE Club .
diff --git a/openpype/lib/terminal.py b/openpype/lib/terminal.py
index ddc917ac4ee..f6072ed209c 100644
--- a/openpype/lib/terminal.py
+++ b/openpype/lib/terminal.py
@@ -49,11 +49,13 @@ def _initialize():
"""
from openpype.lib import env_value_to_bool
- use_colors = env_value_to_bool(
- "OPENPYPE_LOG_NO_COLORS", default=Terminal.use_colors
+ log_no_colors = env_value_to_bool(
+ "OPENPYPE_LOG_NO_COLORS", default=None
)
- if not use_colors:
- Terminal.use_colors = use_colors
+ if log_no_colors is not None:
+ Terminal.use_colors = not log_no_colors
+
+ if not Terminal.use_colors:
Terminal._initialized = True
return
@@ -96,7 +98,7 @@ def _initialize():
r"\*\*\* WRN": _SB + _LY + r"*** WRN" + _RST,
r" \- ": _SB + _LY + r" - " + _RST,
r"\[ ": _SB + _LG + r"[ " + _RST,
- r"\]": _SB + _LG + r"]" + _RST,
+ r" \]": _SB + _LG + r" ]" + _RST,
r"{": _LG + r"{",
r"}": r"}" + _RST,
r"\(": _LY + r"(",
@@ -130,7 +132,7 @@ def _initialize():
def _multiple_replace(text, adict):
"""Replace multiple tokens defined in dict.
- Find and replace all occurances of strings defined in dict is
+ Find and replace all occurrences of strings defined in dict is
supplied string.
Args:
diff --git a/openpype/lib/terminal_splash.py b/openpype/lib/terminal_splash.py
deleted file mode 100644
index 0ba2706a278..00000000000
--- a/openpype/lib/terminal_splash.py
+++ /dev/null
@@ -1,43 +0,0 @@
-# -*- coding: utf-8 -*-
-"""Pype terminal animation."""
-import blessed
-from pathlib import Path
-from time import sleep
-
-NO_TERMINAL = False
-
-try:
- term = blessed.Terminal()
-except AttributeError:
- # this happens when blessed cannot find proper terminal.
- # If so, skip printing ascii art animation.
- NO_TERMINAL = True
-
-
-def play_animation():
- """Play ASCII art Pype animation."""
- if NO_TERMINAL:
- return
- print(term.home + term.clear)
- frame_size = 7
- splash_file = Path(__file__).parent / "splash.txt"
- with splash_file.open("r") as sf:
- animation = sf.readlines()
-
- animation_length = int(len(animation) / frame_size)
- current_frame = 0
- for _ in range(animation_length):
- frame = "".join(
- scanline
- for y, scanline in enumerate(
- animation[current_frame: current_frame + frame_size]
- )
- )
-
- with term.location(0, 0):
- # term.aquamarine3_bold(frame)
- print(f"{term.bold}{term.aquamarine3}{frame}{term.normal}")
-
- sleep(0.02)
- current_frame += frame_size
- print(term.move_y(7))
diff --git a/openpype/lib/transcoding.py b/openpype/lib/transcoding.py
index 3d587e2f29f..51e34312f24 100644
--- a/openpype/lib/transcoding.py
+++ b/openpype/lib/transcoding.py
@@ -1,15 +1,47 @@
import os
import re
import logging
+import json
import collections
import tempfile
+import subprocess
+
+import xml.etree.ElementTree
from .execute import run_subprocess
from .vendor_bin_utils import (
+ get_ffmpeg_tool_path,
get_oiio_tools_path,
- is_oiio_supported
+ is_oiio_supported,
)
+# Max length of string that is supported by ffmpeg
+MAX_FFMPEG_STRING_LEN = 8196
+# Not allowed symbols in attributes for ffmpeg
+NOT_ALLOWED_FFMPEG_CHARS = ("\"", )
+
+# OIIO known xml tags
+STRING_TAGS = {
+ "format"
+}
+INT_TAGS = {
+ "x", "y", "z",
+ "width", "height", "depth",
+ "full_x", "full_y", "full_z",
+ "full_width", "full_height", "full_depth",
+ "tile_width", "tile_height", "tile_depth",
+ "nchannels",
+ "alpha_channel",
+ "z_channel",
+ "deep",
+ "subimages",
+}
+
+XML_CHAR_REF_REGEX_HEX = re.compile(r"&#x?[0-9a-fA-F]+;")
+
+# Regex to parse array attributes
+ARRAY_TYPE_REGEX = re.compile(r"^(int|float|string)\[\d+\]$")
+
def get_transcode_temp_directory():
"""Creates temporary folder for transcoding.
@@ -24,87 +56,243 @@ def get_transcode_temp_directory():
def get_oiio_info_for_input(filepath, logger=None):
- """Call oiiotool to get information about input and return stdout."""
+ """Call oiiotool to get information about input and return stdout.
+
+ Stdout should contain xml format string.
+ """
args = [
- get_oiio_tools_path(), "--info", "-v", filepath
+ get_oiio_tools_path(), "--info", "-v", "-i:infoformat=xml", filepath
]
- return run_subprocess(args, logger=logger)
+ output = run_subprocess(args, logger=logger)
+ output = output.replace("\r\n", "\n")
+
+ xml_started = False
+ lines = []
+ for line in output.split("\n"):
+ if not xml_started:
+ if not line.startswith("<"):
+ continue
+ xml_started = True
+ if xml_started:
+ lines.append(line)
+
+ if not xml_started:
+ raise ValueError(
+ "Failed to read input file \"{}\".\nOutput:\n{}".format(
+ filepath, output
+ )
+ )
+ xml_text = "\n".join(lines)
+ return parse_oiio_xml_output(xml_text, logger=logger)
-def parse_oiio_info(oiio_info):
- """Create an object based on output from oiiotool.
- Removes quotation marks from compression value. Parse channels into
- dictionary - key is channel name value is determined type of channel
- (e.g. 'uint', 'float').
+class RationalToInt:
+ """Rational value stored as division of 2 integers using string."""
+ def __init__(self, string_value):
+ parts = string_value.split("/")
+ top = float(parts[0])
+ bottom = 1.0
+ if len(parts) != 1:
+ bottom = float(parts[1])
- Args:
- oiio_info (str): Output of calling "oiiotool --info -v "
+ self._value = float(top) / float(bottom)
+ self._string_value = string_value
- Returns:
- dict: Loaded data from output.
+ @property
+ def value(self):
+ return self._value
+
+ @property
+ def string_value(self):
+ return self._string_value
+
+ def __format__(self, *args, **kwargs):
+ return self._string_value.__format__(*args, **kwargs)
+
+ def __float__(self):
+ return self._value
+
+ def __str__(self):
+ return self._string_value
+
+ def __repr__(self):
+ return "<{}> {}".format(self.__class__.__name__, self._string_value)
+
+
+def convert_value_by_type_name(value_type, value, logger=None):
+ """Convert value to proper type based on type name.
+
+ In some cases value types have custom python class.
"""
- lines = [
- line.strip()
- for line in oiio_info.split("\n")
- ]
- # Each line should contain information about one key
- # key - value are separated with ": "
- oiio_sep = ": "
- data_map = {}
- for line in lines:
- parts = line.split(oiio_sep)
- if len(parts) < 2:
+ if logger is None:
+ logger = logging.getLogger(__name__)
+
+ # Simple types
+ if value_type == "string":
+ return value
+
+ if value_type == "int":
+ return int(value)
+
+ if value_type == "float":
+ return float(value)
+
+ # Vectors will probably have more types
+ if value_type == "vec2f":
+ return [float(item) for item in value.split(",")]
+
+ # Matrix should be always have square size of element 3x3, 4x4
+ # - are returned as list of lists
+ if value_type == "matrix":
+ output = []
+ current_index = -1
+ parts = value.split(",")
+ parts_len = len(parts)
+ if parts_len == 1:
+ divisor = 1
+ elif parts_len == 4:
+ divisor = 2
+ elif parts_len == 9:
+ divisor = 3
+ elif parts_len == 16:
+ divisor = 4
+ else:
+ logger.info("Unknown matrix resolution {}. Value: \"{}\"".format(
+ parts_len, value
+ ))
+ for part in parts:
+ output.append(float(part))
+ return output
+
+ for idx, item in enumerate(parts):
+ list_index = idx % divisor
+ if list_index > current_index:
+ current_index = list_index
+ output.append([])
+ output[list_index].append(float(item))
+ return output
+
+ if value_type == "rational2i":
+ return RationalToInt(value)
+
+ if value_type == "vector":
+ parts = [part.strip() for part in value.split(",")]
+ output = []
+ for part in parts:
+ if part == "-nan":
+ output.append(None)
+ continue
+ try:
+ part = float(part)
+ except ValueError:
+ pass
+ output.append(part)
+ return output
+
+ if value_type == "timecode":
+ return value
+
+ # Array of other types is converted to list
+ re_result = ARRAY_TYPE_REGEX.findall(value_type)
+ if re_result:
+ array_type = re_result[0]
+ output = []
+ for item in value.split(","):
+ output.append(
+ convert_value_by_type_name(array_type, item, logger=logger)
+ )
+ return output
+
+ logger.info((
+ "MISSING IMPLEMENTATION:"
+ " Unknown attrib type \"{}\". Value: {}"
+ ).format(value_type, value))
+ return value
+
+
+def parse_oiio_xml_output(xml_string, logger=None):
+ """Parse xml output from OIIO info command."""
+ output = {}
+ if not xml_string:
+ return output
+
+ # Fix values with ampresand (lazy fix)
+ # - oiiotool exports invalid xml which ElementTree can't handle
+ # e.g. ""
+ # WARNING: this will affect even valid character entities. If you need
+ # those values correctly, this must take care of valid character ranges.
+ # See https://github.com/pypeclub/OpenPype/pull/2729
+ matches = XML_CHAR_REF_REGEX_HEX.findall(xml_string)
+ for match in matches:
+ new_value = match.replace("&", "&")
+ xml_string = xml_string.replace(match, new_value)
+
+ if logger is None:
+ logger = logging.getLogger("OIIO-xml-parse")
+
+ tree = xml.etree.ElementTree.fromstring(xml_string)
+ attribs = {}
+ output["attribs"] = attribs
+ for child in tree:
+ tag_name = child.tag
+ if tag_name == "attrib":
+ attrib_def = child.attrib
+ value = convert_value_by_type_name(
+ attrib_def["type"], child.text, logger=logger
+ )
+
+ attribs[attrib_def["name"]] = value
continue
- key = parts.pop(0)
- value = oiio_sep.join(parts)
- data_map[key] = value
-
- if "compression" in data_map:
- value = data_map["compression"]
- data_map["compression"] = value.replace("\"", "")
-
- channels_info = {}
- channels_value = data_map.get("channel list") or ""
- if channels_value:
- channels = channels_value.split(", ")
- type_regex = re.compile(r"(?P[^\(]+) \((?P[^\)]+)\)")
- for channel in channels:
- match = type_regex.search(channel)
- if not match:
- channel_name = channel
- channel_type = "uint"
- else:
- channel_name = match.group("name")
- channel_type = match.group("type")
- channels_info[channel_name] = channel_type
- data_map["channels_info"] = channels_info
- return data_map
-
-
-def get_convert_rgb_channels(channels_info):
+
+ # Channels are stored as tex on each child
+ if tag_name == "channelnames":
+ value = []
+ for channel in child:
+ value.append(channel.text)
+
+ # Convert known integer type tags to int
+ elif tag_name in INT_TAGS:
+ value = int(child.text)
+
+ # Keep value of known string tags
+ elif tag_name in STRING_TAGS:
+ value = child.text
+
+ # Keep value as text for unknown tags
+ # - feel free to add more tags
+ else:
+ value = child.text
+ logger.info((
+ "MISSING IMPLEMENTATION:"
+ " Unknown tag \"{}\". Value \"{}\""
+ ).format(tag_name, value))
+
+ output[child.tag] = value
+
+ return output
+
+
+def get_convert_rgb_channels(channel_names):
"""Get first available RGB(A) group from channels info.
## Examples
```
# Ideal situation
- channels_info: {
- "R": ...,
- "G": ...,
- "B": ...,
- "A": ...
+ channels_info: [
+ "R", "G", "B", "A"
}
```
Result will be `("R", "G", "B", "A")`
```
# Not ideal situation
- channels_info: {
- "beauty.red": ...,
- "beuaty.green": ...,
- "beauty.blue": ...,
- "depth.Z": ...
- }
+ channels_info: [
+ "beauty.red",
+ "beuaty.green",
+ "beauty.blue",
+ "depth.Z"
+ ]
```
Result will be `("beauty.red", "beauty.green", "beauty.blue", None)`
@@ -116,7 +304,7 @@ def get_convert_rgb_channels(channels_info):
"""
rgb_by_main_name = collections.defaultdict(dict)
main_name_order = [""]
- for channel_name in channels_info.keys():
+ for channel_name in channel_names:
name_parts = channel_name.split(".")
rgb_part = name_parts.pop(-1).lower()
main_name = ".".join(name_parts)
@@ -166,28 +354,44 @@ def should_convert_for_ffmpeg(src_filepath):
return None
# Load info about info from oiio tool
- oiio_info = get_oiio_info_for_input(src_filepath)
- input_info = parse_oiio_info(oiio_info)
+ input_info = get_oiio_info_for_input(src_filepath)
+ if not input_info:
+ return None
# Check compression
- compression = input_info["compression"]
+ compression = input_info["attribs"].get("compression")
if compression in ("dwaa", "dwab"):
return True
# Check channels
- channels_info = input_info["channels_info"]
- review_channels = get_convert_rgb_channels(channels_info)
+ channel_names = input_info["channelnames"]
+ review_channels = get_convert_rgb_channels(channel_names)
if review_channels is None:
return None
+ for attr_value in input_info["attribs"].values():
+ if not isinstance(attr_value, str):
+ continue
+
+ if len(attr_value) > MAX_FFMPEG_STRING_LEN:
+ return True
+
+ for char in NOT_ALLOWED_FFMPEG_CHARS:
+ if char in attr_value:
+ return True
return False
+# Deprecated since 2022 4 20
+# - Reason - Doesn't convert sequences right way: Can't handle gaps, reuse
+# first frame for all frames and changes filenames when input
+# is sequence.
+# - use 'convert_input_paths_for_ffmpeg' instead
def convert_for_ffmpeg(
first_input_path,
output_dir,
- input_frame_start,
- input_frame_end,
+ input_frame_start=None,
+ input_frame_end=None,
logger=None
):
"""Contert source file to format supported in ffmpeg.
@@ -210,6 +414,12 @@ def convert_for_ffmpeg(
if logger is None:
logger = logging.getLogger(__name__)
+ logger.warning((
+ "DEPRECATED: 'openpype.lib.transcoding.convert_for_ffmpeg' is"
+ " deprecated function of conversion for FFMpeg. Please replace usage"
+ " with 'openpype.lib.transcoding.convert_input_paths_for_ffmpeg'"
+ ))
+
ext = os.path.splitext(first_input_path)[1].lower()
if ext != ".exr":
raise ValueError((
@@ -221,46 +431,547 @@ def convert_for_ffmpeg(
if input_frame_start is not None and input_frame_end is not None:
is_sequence = int(input_frame_end) != int(input_frame_start)
- oiio_info = get_oiio_info_for_input(first_input_path)
- input_info = parse_oiio_info(oiio_info)
+ input_info = get_oiio_info_for_input(first_input_path)
# Change compression only if source compression is "dwaa" or "dwab"
# - they're not supported in ffmpeg
- compression = input_info["compression"]
+ compression = input_info["attribs"].get("compression")
if compression in ("dwaa", "dwab"):
compression = "none"
# Prepare subprocess arguments
oiio_cmd = [
get_oiio_tools_path(),
- "--compression", compression,
- first_input_path
+
+ # Don't add any additional attributes
+ "--nosoftwareattrib",
]
+ # Add input compression if available
+ if compression:
+ oiio_cmd.extend(["--compression", compression])
- channels_info = input_info["channels_info"]
- review_channels = get_convert_rgb_channels(channels_info)
+ # Collect channels to export
+ channel_names = input_info["channelnames"]
+ review_channels = get_convert_rgb_channels(channel_names)
if review_channels is None:
raise ValueError(
"Couldn't find channels that can be used for conversion."
)
red, green, blue, alpha = review_channels
+ input_channels = [red, green, blue]
channels_arg = "R={},G={},B={}".format(red, green, blue)
if alpha is not None:
channels_arg += ",A={}".format(alpha)
- oiio_cmd.append("--ch")
- oiio_cmd.append(channels_arg)
+ input_channels.append(alpha)
+ input_channels_str = ",".join(input_channels)
+
+ oiio_cmd.extend([
+ # Tell oiiotool which channels should be loaded
+ # - other channels are not loaded to memory so helps to avoid memory
+ # leak issues
+ "-i:ch={}".format(input_channels_str), first_input_path,
+ # Tell oiiotool which channels should be put to top stack (and output)
+ "--ch", channels_arg
+ ])
# Add frame definitions to arguments
if is_sequence:
- oiio_cmd.append("--frames")
- oiio_cmd.append("{}-{}".format(input_frame_start, input_frame_end))
+ oiio_cmd.extend([
+ "--frames", "{}-{}".format(input_frame_start, input_frame_end)
+ ])
+
+ for attr_name, attr_value in input_info["attribs"].items():
+ if not isinstance(attr_value, str):
+ continue
+
+ # Remove attributes that have string value longer than allowed length
+ # for ffmpeg or when containt unallowed symbols
+ erase_reason = "Missing reason"
+ erase_attribute = False
+ if len(attr_value) > MAX_FFMPEG_STRING_LEN:
+ erase_reason = "has too long value ({} chars).".format(
+ len(attr_value)
+ )
+ erase_attribute = True
+
+ if not erase_attribute:
+ for char in NOT_ALLOWED_FFMPEG_CHARS:
+ if char in attr_value:
+ erase_attribute = True
+ erase_reason = (
+ "contains unsupported character \"{}\"."
+ ).format(char)
+ break
+
+ if erase_attribute:
+ # Set attribute to empty string
+ logger.info((
+ "Removed attribute \"{}\" from metadata because {}."
+ ).format(attr_name, erase_reason))
+ oiio_cmd.extend(["--eraseattrib", attr_name])
# Add last argument - path to output
- base_file_name = os.path.basename(first_input_path)
- output_path = os.path.join(output_dir, base_file_name)
- oiio_cmd.append("-o")
- oiio_cmd.append(output_path)
+ if is_sequence:
+ ext = os.path.splitext(first_input_path)[1]
+ base_filename = "tmp.%{:0>2}d{}".format(
+ len(str(input_frame_end)), ext
+ )
+ else:
+ base_filename = os.path.basename(first_input_path)
+ output_path = os.path.join(output_dir, base_filename)
+ oiio_cmd.extend([
+ "-o", output_path
+ ])
logger.debug("Conversion command: {}".format(" ".join(oiio_cmd)))
run_subprocess(oiio_cmd, logger=logger)
+
+
+def convert_input_paths_for_ffmpeg(
+ input_paths,
+ output_dir,
+ logger=None
+):
+ """Convert source file to format supported in ffmpeg.
+
+ Currently can convert only exrs. The input filepaths should be files
+ with same type. Information about input is loaded only from first found
+ file.
+
+ Filenames of input files are kept so make sure that output directory
+ is not the same directory as input files have.
+ - This way it can handle gaps and can keep input filenames without handling
+ frame template
+
+ Args:
+ input_paths (str): Paths that should be converted. It is expected that
+ contains single file or image sequence of samy type.
+ output_dir (str): Path to directory where output will be rendered.
+ Must not be same as input's directory.
+ logger (logging.Logger): Logger used for logging.
+
+ Raises:
+ ValueError: If input filepath has extension not supported by function.
+ Currently is supported only ".exr" extension.
+ """
+ if logger is None:
+ logger = logging.getLogger(__name__)
+
+ first_input_path = input_paths[0]
+ ext = os.path.splitext(first_input_path)[1].lower()
+ if ext != ".exr":
+ raise ValueError((
+ "Function 'convert_for_ffmpeg' currently support only"
+ " \".exr\" extension. Got \"{}\"."
+ ).format(ext))
+
+ input_info = get_oiio_info_for_input(first_input_path)
+
+ # Change compression only if source compression is "dwaa" or "dwab"
+ # - they're not supported in ffmpeg
+ compression = input_info["attribs"].get("compression")
+ if compression in ("dwaa", "dwab"):
+ compression = "none"
+
+ # Collect channels to export
+ channel_names = input_info["channelnames"]
+ review_channels = get_convert_rgb_channels(channel_names)
+ if review_channels is None:
+ raise ValueError(
+ "Couldn't find channels that can be used for conversion."
+ )
+
+ red, green, blue, alpha = review_channels
+ input_channels = [red, green, blue]
+ channels_arg = "R={},G={},B={}".format(red, green, blue)
+ if alpha is not None:
+ channels_arg += ",A={}".format(alpha)
+ input_channels.append(alpha)
+ input_channels_str = ",".join(input_channels)
+
+ for input_path in input_paths:
+ # Prepare subprocess arguments
+ oiio_cmd = [
+ get_oiio_tools_path(),
+
+ # Don't add any additional attributes
+ "--nosoftwareattrib",
+ ]
+ # Add input compression if available
+ if compression:
+ oiio_cmd.extend(["--compression", compression])
+
+ oiio_cmd.extend([
+ # Tell oiiotool which channels should be loaded
+ # - other channels are not loaded to memory so helps to
+ # avoid memory leak issues
+ "-i:ch={}".format(input_channels_str), input_path,
+ # Tell oiiotool which channels should be put to top stack
+ # (and output)
+ "--ch", channels_arg
+ ])
+
+ for attr_name, attr_value in input_info["attribs"].items():
+ if not isinstance(attr_value, str):
+ continue
+
+ # Remove attributes that have string value longer than allowed
+ # length for ffmpeg or when containt unallowed symbols
+ erase_reason = "Missing reason"
+ erase_attribute = False
+ if len(attr_value) > MAX_FFMPEG_STRING_LEN:
+ erase_reason = "has too long value ({} chars).".format(
+ len(attr_value)
+ )
+ erase_attribute = True
+
+ if not erase_attribute:
+ for char in NOT_ALLOWED_FFMPEG_CHARS:
+ if char in attr_value:
+ erase_attribute = True
+ erase_reason = (
+ "contains unsupported character \"{}\"."
+ ).format(char)
+ break
+
+ if erase_attribute:
+ # Set attribute to empty string
+ logger.info((
+ "Removed attribute \"{}\" from metadata because {}."
+ ).format(attr_name, erase_reason))
+ oiio_cmd.extend(["--eraseattrib", attr_name])
+
+ # Add last argument - path to output
+ base_filename = os.path.basename(input_path)
+ output_path = os.path.join(output_dir, base_filename)
+ oiio_cmd.extend([
+ "-o", output_path
+ ])
+
+ logger.debug("Conversion command: {}".format(" ".join(oiio_cmd)))
+ run_subprocess(oiio_cmd, logger=logger)
+
+
+# FFMPEG functions
+def get_ffprobe_data(path_to_file, logger=None):
+ """Load data about entered filepath via ffprobe.
+
+ Args:
+ path_to_file (str): absolute path
+ logger (logging.Logger): injected logger, if empty new is created
+ """
+ if not logger:
+ logger = logging.getLogger(__name__)
+ logger.info(
+ "Getting information about input \"{}\".".format(path_to_file)
+ )
+ args = [
+ get_ffmpeg_tool_path("ffprobe"),
+ "-hide_banner",
+ "-loglevel", "fatal",
+ "-show_error",
+ "-show_format",
+ "-show_streams",
+ "-show_programs",
+ "-show_chapters",
+ "-show_private_data",
+ "-print_format", "json",
+ path_to_file
+ ]
+
+ logger.debug("FFprobe command: {}".format(
+ subprocess.list2cmdline(args)
+ ))
+ popen = subprocess.Popen(
+ args,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE
+ )
+
+ popen_stdout, popen_stderr = popen.communicate()
+ if popen_stdout:
+ logger.debug("FFprobe stdout:\n{}".format(
+ popen_stdout.decode("utf-8")
+ ))
+
+ if popen_stderr:
+ logger.warning("FFprobe stderr:\n{}".format(
+ popen_stderr.decode("utf-8")
+ ))
+
+ return json.loads(popen_stdout)
+
+
+def get_ffprobe_streams(path_to_file, logger=None):
+ """Load streams from entered filepath via ffprobe.
+
+ Args:
+ path_to_file (str): absolute path
+ logger (logging.Logger): injected logger, if empty new is created
+ """
+ return get_ffprobe_data(path_to_file, logger)["streams"]
+
+
+def get_ffmpeg_format_args(ffprobe_data, source_ffmpeg_cmd=None):
+ """Copy format from input metadata for output.
+
+ Args:
+ ffprobe_data(dict): Data received from ffprobe.
+ source_ffmpeg_cmd(str): Command that created input if available.
+ """
+ input_format = ffprobe_data.get("format") or {}
+ if input_format.get("format_name") == "mxf":
+ return _ffmpeg_mxf_format_args(ffprobe_data, source_ffmpeg_cmd)
+ return []
+
+
+def _ffmpeg_mxf_format_args(ffprobe_data, source_ffmpeg_cmd):
+ input_format = ffprobe_data["format"]
+ format_tags = input_format.get("tags") or {}
+ operational_pattern_ul = format_tags.get("operational_pattern_ul") or ""
+ output = []
+ if operational_pattern_ul == "060e2b34.04010102.0d010201.10030000":
+ output.extend(["-f", "mxf_opatom"])
+ return output
+
+
+def get_ffmpeg_codec_args(ffprobe_data, source_ffmpeg_cmd=None, logger=None):
+ """Copy codec from input metadata for output.
+
+ Args:
+ ffprobe_data(dict): Data received from ffprobe.
+ source_ffmpeg_cmd(str): Command that created input if available.
+ """
+ if logger is None:
+ logger = logging.getLogger(__name__)
+
+ video_stream = None
+ no_audio_stream = None
+ for stream in ffprobe_data["streams"]:
+ codec_type = stream["codec_type"]
+ if codec_type == "video":
+ video_stream = stream
+ break
+ elif no_audio_stream is None and codec_type != "audio":
+ no_audio_stream = stream
+
+ if video_stream is None:
+ if no_audio_stream is None:
+ logger.warning(
+ "Couldn't find stream that is not an audio file."
+ )
+ return []
+ logger.info(
+ "Didn't find video stream. Using first non audio stream."
+ )
+ video_stream = no_audio_stream
+
+ codec_name = video_stream.get("codec_name")
+ # Codec "prores"
+ if codec_name == "prores":
+ return _ffmpeg_prores_codec_args(video_stream, source_ffmpeg_cmd)
+
+ # Codec "h264"
+ if codec_name == "h264":
+ return _ffmpeg_h264_codec_args(video_stream, source_ffmpeg_cmd)
+
+ # Coded DNxHD
+ if codec_name == "dnxhd":
+ return _ffmpeg_dnxhd_codec_args(video_stream, source_ffmpeg_cmd)
+
+ output = []
+ if codec_name:
+ output.extend(["-codec:v", codec_name])
+
+ bit_rate = video_stream.get("bit_rate")
+ if bit_rate:
+ output.extend(["-b:v", bit_rate])
+
+ pix_fmt = video_stream.get("pix_fmt")
+ if pix_fmt:
+ output.extend(["-pix_fmt", pix_fmt])
+
+ output.extend(["-g", "1"])
+
+ return output
+
+
+def _ffmpeg_prores_codec_args(stream_data, source_ffmpeg_cmd):
+ output = []
+
+ tags = stream_data.get("tags") or {}
+ encoder = tags.get("encoder") or ""
+ if encoder.endswith("prores_ks"):
+ codec_name = "prores_ks"
+
+ elif encoder.endswith("prores_aw"):
+ codec_name = "prores_aw"
+
+ else:
+ codec_name = "prores"
+
+ output.extend(["-codec:v", codec_name])
+
+ pix_fmt = stream_data.get("pix_fmt")
+ if pix_fmt:
+ output.extend(["-pix_fmt", pix_fmt])
+
+ # Rest of arguments is prores_kw specific
+ if codec_name == "prores_ks":
+ codec_tag_to_profile_map = {
+ "apco": "proxy",
+ "apcs": "lt",
+ "apcn": "standard",
+ "apch": "hq",
+ "ap4h": "4444",
+ "ap4x": "4444xq"
+ }
+ codec_tag_str = stream_data.get("codec_tag_string")
+ if codec_tag_str:
+ profile = codec_tag_to_profile_map.get(codec_tag_str)
+ if profile:
+ output.extend(["-profile:v", profile])
+
+ return output
+
+
+def _ffmpeg_h264_codec_args(stream_data, source_ffmpeg_cmd):
+ output = ["-codec:v", "h264"]
+
+ # Use arguments from source if are available source arguments
+ if source_ffmpeg_cmd:
+ copy_args = (
+ "-crf",
+ "-b:v", "-vb",
+ "-minrate", "-minrate:",
+ "-maxrate", "-maxrate:",
+ "-bufsize", "-bufsize:"
+ )
+ args = source_ffmpeg_cmd.split(" ")
+ for idx, arg in enumerate(args):
+ if arg in copy_args:
+ output.extend([arg, args[idx + 1]])
+
+ pix_fmt = stream_data.get("pix_fmt")
+ if pix_fmt:
+ output.extend(["-pix_fmt", pix_fmt])
+
+ output.extend(["-intra"])
+ output.extend(["-g", "1"])
+
+ return output
+
+
+def _ffmpeg_dnxhd_codec_args(stream_data, source_ffmpeg_cmd):
+ output = ["-codec:v", "dnxhd"]
+
+ # Use source profile (profiles in metadata are not usable in args directly)
+ profile = stream_data.get("profile") or ""
+ # Lower profile and replace space with underscore
+ cleaned_profile = profile.lower().replace(" ", "_")
+
+ # TODO validate this statement
+ # Looks like using 'dnxhd' profile must have set bit rate and in that case
+ # should be used bitrate from source.
+ # - related attributes 'bit_rate_defined', 'bit_rate_must_be_defined'
+ bit_rate_must_be_defined = True
+ dnx_profiles = {
+ "dnxhd",
+ "dnxhr_lb",
+ "dnxhr_sq",
+ "dnxhr_hq",
+ "dnxhr_hqx",
+ "dnxhr_444"
+ }
+ if cleaned_profile in dnx_profiles:
+ if cleaned_profile != "dnxhd":
+ bit_rate_must_be_defined = False
+ output.extend(["-profile:v", cleaned_profile])
+
+ pix_fmt = stream_data.get("pix_fmt")
+ if pix_fmt:
+ output.extend(["-pix_fmt", pix_fmt])
+
+ # Use arguments from source if are available source arguments
+ bit_rate_defined = False
+ if source_ffmpeg_cmd:
+ # Define bitrate arguments
+ bit_rate_args = ("-b:v", "-vb",)
+ # Seprate the two variables in case something else should be copied
+ # from source command
+ copy_args = []
+ copy_args.extend(bit_rate_args)
+
+ args = source_ffmpeg_cmd.split(" ")
+ for idx, arg in enumerate(args):
+ if arg in copy_args:
+ if arg in bit_rate_args:
+ bit_rate_defined = True
+ output.extend([arg, args[idx + 1]])
+
+ # Add bitrate if needed
+ if bit_rate_must_be_defined and not bit_rate_defined:
+ src_bit_rate = stream_data.get("bit_rate")
+ if src_bit_rate:
+ output.extend(["-b:v", src_bit_rate])
+
+ output.extend(["-g", "1"])
+ return output
+
+
+def convert_ffprobe_fps_value(str_value):
+ """Returns (str) value of fps from ffprobe frame format (120/1)"""
+ if str_value == "0/0":
+ print("WARNING: Source has \"r_frame_rate\" value set to \"0/0\".")
+ return "Unknown"
+
+ items = str_value.split("/")
+ if len(items) == 1:
+ fps = float(items[0])
+
+ elif len(items) == 2:
+ fps = float(items[0]) / float(items[1])
+
+ # Check if fps is integer or float number
+ if int(fps) == fps:
+ fps = int(fps)
+
+ return str(fps)
+
+
+def convert_ffprobe_fps_to_float(value):
+ """Convert string value of frame rate to float.
+
+ Copy of 'convert_ffprobe_fps_value' which raises exceptions on invalid
+ value, does not convert value to string and does not return "Unknown"
+ string.
+
+ Args:
+ value (str): Value to be converted.
+
+ Returns:
+ Float: Converted frame rate in float. If divisor in value is '0' then
+ '0.0' is returned.
+
+ Raises:
+ ValueError: Passed value is invalid for conversion.
+ """
+
+ if not value:
+ raise ValueError("Got empty value.")
+
+ items = value.split("/")
+ if len(items) == 1:
+ return float(items[0])
+
+ if len(items) > 2:
+ raise ValueError((
+ "FPS expression contains multiple dividers \"{}\"."
+ ).format(value))
+
+ dividend = float(items.pop(0))
+ divisor = float(items.pop(0))
+ if divisor == 0.0:
+ return 0.0
+ return dividend / divisor
diff --git a/openpype/lib/usdlib.py b/openpype/lib/usdlib.py
index 3ae7430c7b4..20703ee3082 100644
--- a/openpype/lib/usdlib.py
+++ b/openpype/lib/usdlib.py
@@ -8,7 +8,8 @@
# Allow to fall back on Multiverse 6.3.0+ pxr usd library
from mvpxr import Usd, UsdGeom, Sdf, Kind
-from avalon import io, api
+from openpype.client import get_project, get_asset_by_name
+from openpype.pipeline import legacy_io, Anatomy
log = logging.getLogger(__name__)
@@ -125,7 +126,8 @@ def create_model(filename, asset, variant_subsets):
"""
- asset_doc = io.find_one({"name": asset, "type": "asset"})
+ project_name = legacy_io.active_project()
+ asset_doc = get_asset_by_name(project_name, asset)
assert asset_doc, "Asset not found: %s" % asset
variants = []
@@ -175,7 +177,8 @@ def create_shade(filename, asset, variant_subsets):
"""
- asset_doc = io.find_one({"name": asset, "type": "asset"})
+ project_name = legacy_io.active_project()
+ asset_doc = get_asset_by_name(project_name, asset)
assert asset_doc, "Asset not found: %s" % asset
variants = []
@@ -210,7 +213,8 @@ def create_shade_variation(filename, asset, model_variant, shade_variants):
"""
- asset_doc = io.find_one({"name": asset, "type": "asset"})
+ project_name = legacy_io.active_project()
+ asset_doc = get_asset_by_name(project_name, asset)
assert asset_doc, "Asset not found: %s" % asset
variants = []
@@ -310,22 +314,25 @@ def get_usd_master_path(asset, subset, representation):
"""
- project = io.find_one(
- {"type": "project"}, projection={"config.template.publish": True}
+ project_name = legacy_io.active_project()
+ anatomy = Anatomy(project_name)
+ project_doc = get_project(
+ project_name,
+ fields=["name", "data.code"]
)
- template = project["config"]["template"]["publish"]
- if isinstance(asset, dict) and "silo" in asset and "name" in asset:
+ if isinstance(asset, dict) and "name" in asset:
# Allow explicitly passing asset document
asset_doc = asset
else:
- asset_doc = io.find_one({"name": asset, "type": "asset"})
-
- path = template.format(
- **{
- "root": api.registered_root(),
- "project": api.Session["AVALON_PROJECT"],
- "silo": asset_doc["silo"],
+ asset_doc = get_asset_by_name(project_name, asset, fields=["name"])
+
+ formatted_result = anatomy.format(
+ {
+ "project": {
+ "name": project_name,
+ "code": project_doc.get("data", {}).get("code")
+ },
"asset": asset_doc["name"],
"subset": subset,
"representation": representation,
@@ -333,6 +340,7 @@ def get_usd_master_path(asset, subset, representation):
}
)
+ path = formatted_result["publish"]["path"]
# Remove the version folder
subset_folder = os.path.dirname(os.path.dirname(path))
master_folder = os.path.join(subset_folder, "master")
diff --git a/openpype/lib/vendor_bin_utils.py b/openpype/lib/vendor_bin_utils.py
index a5d4153b2a4..e5ab2872a0e 100644
--- a/openpype/lib/vendor_bin_utils.py
+++ b/openpype/lib/vendor_bin_utils.py
@@ -1,11 +1,87 @@
import os
import logging
-import json
import platform
-import subprocess
-import distutils
-log = logging.getLogger("FFmpeg utils")
+log = logging.getLogger("Vendor utils")
+
+
+def is_file_executable(filepath):
+ """Filepath lead to executable file.
+
+ Args:
+ filepath(str): Full path to file.
+ """
+ if not filepath:
+ return False
+
+ if os.path.isfile(filepath):
+ if os.access(filepath, os.X_OK):
+ return True
+
+ log.info(
+ "Filepath is not available for execution \"{}\"".format(filepath)
+ )
+ return False
+
+
+def find_executable(executable):
+ """Find full path to executable.
+
+ Also tries additional extensions if passed executable does not contain one.
+
+ Paths where it is looked for executable is defined by 'PATH' environment
+ variable, 'os.confstr("CS_PATH")' or 'os.defpath'.
+
+ Args:
+ executable(str): Name of executable with or without extension. Can be
+ path to file.
+
+ Returns:
+ str: Full path to executable with extension (is file).
+ None: When the executable was not found.
+ """
+ # Skip if passed path is file
+ if is_file_executable(executable):
+ return executable
+
+ low_platform = platform.system().lower()
+ _, ext = os.path.splitext(executable)
+
+ # Prepare variants for which it will be looked
+ variants = [executable]
+ # Add other extension variants only if passed executable does not have one
+ if not ext:
+ if low_platform == "windows":
+ exts = [".exe", ".ps1", ".bat"]
+ for ext in os.getenv("PATHEXT", "").split(os.pathsep):
+ ext = ext.lower()
+ if ext and ext not in exts:
+ exts.append(ext)
+ else:
+ exts = [".sh"]
+
+ for ext in exts:
+ variant = executable + ext
+ if is_file_executable(variant):
+ return variant
+ variants.append(variant)
+
+ # Get paths where to look for executable
+ path_str = os.environ.get("PATH", None)
+ if path_str is None:
+ if hasattr(os, "confstr"):
+ path_str = os.confstr("CS_PATH")
+ elif hasattr(os, "defpath"):
+ path_str = os.defpath
+
+ if path_str:
+ paths = path_str.split(os.pathsep)
+ for path in paths:
+ for variant in variants:
+ filepath = os.path.abspath(os.path.join(path, variant))
+ if is_file_executable(filepath):
+ return filepath
+ return None
def get_vendor_bin_path(bin_app):
@@ -34,12 +110,17 @@ def get_vendor_bin_path(bin_app):
def get_oiio_tools_path(tool="oiiotool"):
"""Path to vendorized OpenImageIO tool executables.
+ On Window it adds .exe extension if missing from tool argument.
+
Args:
tool (string): Tool name (oiiotool, maketx, ...).
Default is "oiiotool".
"""
+
oiio_dir = get_vendor_bin_path("oiio")
- return os.path.join(oiio_dir, tool)
+ if platform.system().lower() == "linux":
+ oiio_dir = os.path.join(oiio_dir, "bin")
+ return find_executable(os.path.join(oiio_dir, tool))
def get_ffmpeg_tool_path(tool="ffmpeg"):
@@ -55,57 +136,7 @@ def get_ffmpeg_tool_path(tool="ffmpeg"):
ffmpeg_dir = get_vendor_bin_path("ffmpeg")
if platform.system().lower() == "windows":
ffmpeg_dir = os.path.join(ffmpeg_dir, "bin")
- return os.path.join(ffmpeg_dir, tool)
-
-
-def ffprobe_streams(path_to_file, logger=None):
- """Load streams from entered filepath via ffprobe.
-
- Args:
- path_to_file (str): absolute path
- logger (logging.getLogger): injected logger, if empty new is created
-
- """
- if not logger:
- logger = log
- logger.info(
- "Getting information about input \"{}\".".format(path_to_file)
- )
- args = [
- get_ffmpeg_tool_path("ffprobe"),
- "-hide_banner",
- "-loglevel", "fatal",
- "-show_error",
- "-show_format",
- "-show_streams",
- "-show_programs",
- "-show_chapters",
- "-show_private_data",
- "-print_format", "json",
- path_to_file
- ]
-
- logger.debug("FFprobe command: {}".format(
- subprocess.list2cmdline(args)
- ))
- popen = subprocess.Popen(
- args,
- stdout=subprocess.PIPE,
- stderr=subprocess.PIPE
- )
-
- popen_stdout, popen_stderr = popen.communicate()
- if popen_stdout:
- logger.debug("FFprobe stdout:\n{}".format(
- popen_stdout.decode("utf-8")
- ))
-
- if popen_stderr:
- logger.warning("FFprobe stderr:\n{}".format(
- popen_stderr.decode("utf-8")
- ))
-
- return json.loads(popen_stdout)["streams"]
+ return find_executable(os.path.join(ffmpeg_dir, tool))
def is_oiio_supported():
@@ -116,7 +147,7 @@ def is_oiio_supported():
"""
loaded_path = oiio_path = get_oiio_tools_path()
if oiio_path:
- oiio_path = distutils.spawn.find_executable(oiio_path)
+ oiio_path = find_executable(oiio_path)
if not oiio_path:
log.debug("OIIOTool is not configured or not present at {}".format(
diff --git a/openpype/modules/__init__.py b/openpype/modules/__init__.py
index 68b5f6c247b..02e7dc13abc 100644
--- a/openpype/modules/__init__.py
+++ b/openpype/modules/__init__.py
@@ -2,7 +2,6 @@
from .base import (
OpenPypeModule,
OpenPypeAddOn,
- OpenPypeInterface,
load_modules,
@@ -20,7 +19,6 @@
__all__ = (
"OpenPypeModule",
"OpenPypeAddOn",
- "OpenPypeInterface",
"load_modules",
diff --git a/openpype/modules/avalon_apps/avalon_app.py b/openpype/modules/avalon_apps/avalon_app.py
index 9e650a097e2..1d21de129bf 100644
--- a/openpype/modules/avalon_apps/avalon_app.py
+++ b/openpype/modules/avalon_apps/avalon_app.py
@@ -1,5 +1,5 @@
import os
-import openpype
+
from openpype.modules import OpenPypeModule
from openpype_interfaces import ITrayModule
@@ -13,14 +13,6 @@ def initialize(self, modules_settings):
avalon_settings = modules_settings[self.name]
- # Check if environment is already set
- avalon_mongo_url = os.environ.get("AVALON_MONGO")
- if not avalon_mongo_url:
- avalon_mongo_url = avalon_settings["AVALON_MONGO"]
- # Use pype mongo if Avalon's mongo not defined
- if not avalon_mongo_url:
- avalon_mongo_url = os.environ["OPENPYPE_MONGO"]
-
thumbnail_root = os.environ.get("AVALON_THUMBNAIL_ROOT")
if not thumbnail_root:
thumbnail_root = avalon_settings["AVALON_THUMBNAIL_ROOT"]
@@ -31,11 +23,11 @@ def initialize(self, modules_settings):
avalon_mongo_timeout = avalon_settings["AVALON_TIMEOUT"]
self.thumbnail_root = thumbnail_root
- self.avalon_mongo_url = avalon_mongo_url
self.avalon_mongo_timeout = avalon_mongo_timeout
# Tray attributes
- self.libraryloader = None
+ self._library_loader_imported = None
+ self._library_loader_window = None
self.rest_api_obj = None
def get_global_environments(self):
@@ -50,13 +42,11 @@ def get_global_environments(self):
def tray_init(self):
# Add library tool
+ self._library_loader_imported = False
try:
from openpype.tools.libraryloader import LibraryLoaderWindow
- self.libraryloader = LibraryLoaderWindow(
- show_projects=True,
- show_libraries=True
- )
+ self._library_loader_imported = True
except Exception:
self.log.warning(
"Couldn't load Library loader tool for tray.",
@@ -65,7 +55,7 @@ def tray_init(self):
# Definition of Tray menu
def tray_menu(self, tray_menu):
- if self.libraryloader is None:
+ if not self._library_loader_imported:
return
from Qt import QtWidgets
@@ -85,17 +75,31 @@ def tray_exit(self, *_a, **_kw):
return
def show_library_loader(self):
- if self.libraryloader is None:
- return
+ if self._library_loader_window is None:
+ from Qt import QtCore
+ from openpype.tools.libraryloader import LibraryLoaderWindow
+ from openpype.pipeline import install_openpype_plugins
+
+ libraryloader = LibraryLoaderWindow(
+ show_projects=True,
+ show_libraries=True
+ )
+ # Remove always on top flag for tray
+ window_flags = libraryloader.windowFlags()
+ if window_flags | QtCore.Qt.WindowStaysOnTopHint:
+ window_flags ^= QtCore.Qt.WindowStaysOnTopHint
+ libraryloader.setWindowFlags(window_flags)
+ self._library_loader_window = libraryloader
+
+ install_openpype_plugins()
- self.libraryloader.show()
+ self._library_loader_window.show()
# Raise and activate the window
# for MacOS
- self.libraryloader.raise_()
+ self._library_loader_window.raise_()
# for Windows
- self.libraryloader.activateWindow()
- self.libraryloader.refresh()
+ self._library_loader_window.activateWindow()
# Webserver module implementation
def webserver_initialization(self, server_manager):
diff --git a/openpype/modules/avalon_apps/rest_api.py b/openpype/modules/avalon_apps/rest_api.py
index 533050fc0c6..a52ce1b6df9 100644
--- a/openpype/modules/avalon_apps/rest_api.py
+++ b/openpype/modules/avalon_apps/rest_api.py
@@ -1,4 +1,3 @@
-import os
import json
import datetime
@@ -6,7 +5,12 @@
from aiohttp.web_response import Response
-from avalon.api import AvalonMongoDB
+from openpype.client import (
+ get_projects,
+ get_project,
+ get_assets,
+ get_asset_by_name,
+)
from openpype_modules.webserver.base_routes import RestApiEndpoint
@@ -15,19 +19,13 @@ def __init__(self, resource):
self.resource = resource
super(_RestApiEndpoint, self).__init__()
- @property
- def dbcon(self):
- return self.resource.dbcon
-
class AvalonProjectsEndpoint(_RestApiEndpoint):
async def get(self) -> Response:
- output = []
- for project_name in self.dbcon.database.collection_names():
- project_doc = self.dbcon.database[project_name].find_one({
- "type": "project"
- })
- output.append(project_doc)
+ output = [
+ project_doc
+ for project_doc in get_projects()
+ ]
return Response(
status=200,
body=self.resource.encode(output),
@@ -37,9 +35,7 @@ async def get(self) -> Response:
class AvalonProjectEndpoint(_RestApiEndpoint):
async def get(self, project_name) -> Response:
- project_doc = self.dbcon.database[project_name].find_one({
- "type": "project"
- })
+ project_doc = get_project(project_name)
if project_doc:
return Response(
status=200,
@@ -54,9 +50,7 @@ async def get(self, project_name) -> Response:
class AvalonAssetsEndpoint(_RestApiEndpoint):
async def get(self, project_name) -> Response:
- asset_docs = list(self.dbcon.database[project_name].find({
- "type": "asset"
- }))
+ asset_docs = list(get_assets(project_name))
return Response(
status=200,
body=self.resource.encode(asset_docs),
@@ -66,10 +60,7 @@ async def get(self, project_name) -> Response:
class AvalonAssetEndpoint(_RestApiEndpoint):
async def get(self, project_name, asset_name) -> Response:
- asset_doc = self.dbcon.database[project_name].find_one({
- "type": "asset",
- "name": asset_name
- })
+ asset_doc = get_asset_by_name(project_name, asset_name)
if asset_doc:
return Response(
status=200,
@@ -89,9 +80,6 @@ def __init__(self, avalon_module, server_manager):
self.module = avalon_module
self.server_manager = server_manager
- self.dbcon = AvalonMongoDB()
- self.dbcon.install()
-
self.prefix = "/avalon"
self.endpoint_defs = (
diff --git a/openpype/modules/base.py b/openpype/modules/base.py
index b5c491a1c00..09aea50424f 100644
--- a/openpype/modules/base.py
+++ b/openpype/modules/base.py
@@ -13,7 +13,6 @@
from abc import ABCMeta, abstractmethod
import six
-import openpype
from openpype.settings import (
get_system_settings,
SYSTEM_SETTINGS_KEY,
@@ -26,22 +25,32 @@
get_studio_system_settings_overrides,
load_json_file
)
-from openpype.lib import PypeLogger
-
-
-DEFAULT_OPENPYPE_MODULES = (
- "avalon_apps",
- "clockify",
- "log_viewer",
- "muster",
- "python_console_interpreter",
- "slack",
- "webserver",
- "launcher_action",
- "project_manager_action",
- "settings_action",
- "standalonepublish_action",
- "job_queue",
+
+from openpype.lib import (
+ Logger,
+ import_filepath,
+ import_module_from_dirpath
+)
+
+from .interfaces import (
+ OpenPypeInterface,
+ IPluginPaths,
+ IHostAddon,
+ ITrayModule,
+ ITrayService
+)
+
+# Files that will be always ignored on modules import
+IGNORED_FILENAMES = (
+ "__pycache__",
+)
+# Files ignored on modules import from "./openpype/modules"
+IGNORED_DEFAULT_FILENAMES = (
+ "__init__.py",
+ "base.py",
+ "interfaces.py",
+ "example_addons",
+ "default_modules",
)
@@ -52,9 +61,11 @@ class _ModuleClass(object):
Object of this class can be stored to `sys.modules` and used for storing
dynamically imported modules.
"""
+
def __init__(self, name):
# Call setattr on super class
super(_ModuleClass, self).__setattr__("name", name)
+ super(_ModuleClass, self).__setattr__("__name__", name)
# Where modules and interfaces are stored
super(_ModuleClass, self).__setattr__("__attributes__", dict())
@@ -66,7 +77,7 @@ def __getattr__(self, attr_name):
if attr_name not in self.__attributes__:
if attr_name in ("__path__", "__file__"):
return None
- raise ImportError("No module named {}.{}".format(
+ raise AttributeError("'{}' has not attribute '{}'".format(
self.name, attr_name
))
return self.__attributes__[attr_name]
@@ -94,7 +105,7 @@ def __getitem__(self, key):
def log(self):
if self._log is None:
super(_ModuleClass, self).__setattr__(
- "_log", PypeLogger.get_logger(self.name)
+ "_log", Logger.get_logger(self.name)
)
return self._log
@@ -118,12 +129,13 @@ class _InterfacesClass(_ModuleClass):
- this is because interfaces must be available even if are missing
implementation
"""
+
def __getattr__(self, attr_name):
if attr_name not in self.__attributes__:
if attr_name in ("__path__", "__file__"):
return None
- raise ImportError((
+ raise AttributeError((
"cannot import name '{}' from 'openpype_interfaces'"
).format(attr_name))
@@ -139,9 +151,16 @@ class _LoadCache:
def get_default_modules_dir():
"""Path to default OpenPype modules."""
- current_dir = os.path.abspath(os.path.dirname(__file__))
- return os.path.join(current_dir, "default_modules")
+ current_dir = os.path.dirname(os.path.abspath(__file__))
+
+ output = []
+ for folder_name in ("default_modules", ):
+ path = os.path.join(current_dir, folder_name)
+ if os.path.exists(path) and os.path.isdir(path):
+ output.append(path)
+
+ return output
def get_dynamic_modules_dirs():
@@ -179,7 +198,7 @@ def get_dynamic_modules_dirs():
def get_module_dirs():
"""List of paths where OpenPype modules can be found."""
_dirpaths = []
- _dirpaths.append(get_default_modules_dir())
+ _dirpaths.extend(get_default_modules_dir())
_dirpaths.extend(get_dynamic_modules_dirs())
dirpaths = []
@@ -217,8 +236,6 @@ def load_interfaces(force=False):
def _load_interfaces():
# Key under which will be modules imported in `sys.modules`
- from openpype.lib import import_filepath
-
modules_key = "openpype_interfaces"
sys.modules[modules_key] = openpype_interfaces = (
@@ -273,100 +290,111 @@ def load_modules(force=False):
def _load_modules():
- # Import helper functions from lib
- from openpype.lib import (
- import_filepath,
- import_module_from_dirpath
- )
-
# Key under which will be modules imported in `sys.modules`
modules_key = "openpype_modules"
# Change `sys.modules`
sys.modules[modules_key] = openpype_modules = _ModuleClass(modules_key)
- log = PypeLogger.get_logger("ModulesLoader")
-
- # Import default modules imported from 'openpype.modules'
- for default_module_name in DEFAULT_OPENPYPE_MODULES:
- try:
- import_str = "openpype.modules.{}".format(default_module_name)
- new_import_str = "{}.{}".format(modules_key, default_module_name)
- default_module = __import__(import_str, fromlist=("", ))
- sys.modules[new_import_str] = default_module
- setattr(openpype_modules, default_module_name, default_module)
-
- except Exception:
- msg = (
- "Failed to import default module '{}'."
- ).format(default_module_name)
- log.error(msg, exc_info=True)
+ log = Logger.get_logger("ModulesLoader")
# Look for OpenPype modules in paths defined with `get_module_dirs`
# - dynamically imported OpenPype modules and addons
- dirpaths = get_module_dirs()
- for dirpath in dirpaths:
+ module_dirs = get_module_dirs()
+ # Add current directory at first place
+ # - has small differences in import logic
+ current_dir = os.path.abspath(os.path.dirname(__file__))
+ hosts_dir = os.path.join(os.path.dirname(current_dir), "hosts")
+ module_dirs.insert(0, hosts_dir)
+ module_dirs.insert(0, current_dir)
+
+ processed_paths = set()
+ for dirpath in module_dirs:
+ # Skip already processed paths
+ if dirpath in processed_paths:
+ continue
+ processed_paths.add(dirpath)
+
if not os.path.exists(dirpath):
log.warning((
"Could not find path when loading OpenPype modules \"{}\""
).format(dirpath))
continue
+ is_in_current_dir = dirpath == current_dir
+ is_in_host_dir = dirpath == hosts_dir
for filename in os.listdir(dirpath):
# Ignore filenames
- if filename in ("__pycache__", ):
+ if filename in IGNORED_FILENAMES:
+ continue
+
+ if (
+ is_in_current_dir
+ and filename in IGNORED_DEFAULT_FILENAMES
+ ):
continue
fullpath = os.path.join(dirpath, filename)
basename, ext = os.path.splitext(filename)
+ # Validations
+ if os.path.isdir(fullpath):
+ # Check existence of init file
+ init_path = os.path.join(fullpath, "__init__.py")
+ if not os.path.exists(init_path):
+ log.debug((
+ "Module directory does not contain __init__.py"
+ " file {}"
+ ).format(fullpath))
+ continue
+
+ elif ext not in (".py", ):
+ continue
+
# TODO add more logic how to define if folder is module or not
# - check manifest and content of manifest
try:
- if os.path.isdir(fullpath):
- # Module without init file can't be used as OpenPype module
- # because the module class could not be imported
- init_file = os.path.join(fullpath, "__init__.py")
- if not os.path.exists(init_file):
- log.info((
- "Skipping module directory because of"
- " missing \"__init__.py\" file. \"{}\""
- ).format(fullpath))
- continue
+ # Don't import dynamically current directory modules
+ if is_in_current_dir:
+ import_str = "openpype.modules.{}".format(basename)
+ new_import_str = "{}.{}".format(modules_key, basename)
+ default_module = __import__(import_str, fromlist=("", ))
+ sys.modules[new_import_str] = default_module
+ setattr(openpype_modules, basename, default_module)
+
+ elif is_in_host_dir:
+ import_str = "openpype.hosts.{}".format(basename)
+ new_import_str = "{}.{}".format(modules_key, basename)
+ # Until all hosts are converted to be able use them as
+ # modules is this error check needed
+ try:
+ default_module = __import__(
+ import_str, fromlist=("", )
+ )
+ sys.modules[new_import_str] = default_module
+ setattr(openpype_modules, basename, default_module)
+
+ except Exception:
+ log.warning(
+ "Failed to import host folder {}".format(basename),
+ exc_info=True
+ )
+
+ elif os.path.isdir(fullpath):
import_module_from_dirpath(dirpath, filename, modules_key)
- elif ext in (".py", ):
+ else:
module = import_filepath(fullpath)
setattr(openpype_modules, basename, module)
except Exception:
- log.error(
- "Failed to import '{}'.".format(fullpath),
- exc_info=True
- )
-
-
-class _OpenPypeInterfaceMeta(ABCMeta):
- """OpenPypeInterface meta class to print proper string."""
- def __str__(self):
- return "<'OpenPypeInterface.{}'>".format(self.__name__)
-
- def __repr__(self):
- return str(self)
-
-
-@six.add_metaclass(_OpenPypeInterfaceMeta)
-class OpenPypeInterface:
- """Base class of Interface that can be used as Mixin with abstract parts.
-
- This is way how OpenPype module or addon can tell that has implementation
- for specific part or for other module/addon.
-
- Child classes of OpenPypeInterface may be used as mixin in different
- OpenPype modules which means they have to have implemented methods defined
- in the interface. By default interface does not have any abstract parts.
- """
- pass
+ if is_in_current_dir:
+ msg = "Failed to import default module '{}'.".format(
+ basename
+ )
+ else:
+ msg = "Failed to import module '{}'.".format(fullpath)
+ log.error(msg, exc_info=True)
@six.add_metaclass(ABCMeta)
@@ -393,7 +421,7 @@ def name(self):
def __init__(self, manager, settings):
self.manager = manager
- self.log = PypeLogger.get_logger(self.name)
+ self.log = Logger.get_logger(self.name)
self.initialize(settings)
@@ -410,10 +438,12 @@ def initialize(self, module_settings):
It is not recommended to override __init__ that's why specific method
was implemented.
"""
+
pass
def connect_with_modules(self, enabled_modules):
"""Connect with other enabled modules."""
+
pass
def get_global_environments(self):
@@ -421,8 +451,41 @@ def get_global_environments(self):
Environment variables that can be get only from system settings.
"""
+
return {}
+ def modify_application_launch_arguments(self, application, env):
+ """Give option to modify launch environments before application launch.
+
+ Implementation is optional. To change environments modify passed
+ dictionary of environments.
+
+ Args:
+ application (Application): Application that is launched.
+ env (dict): Current environemnt variables.
+ """
+
+ pass
+
+ def on_host_install(self, host, host_name, project_name):
+ """Host was installed which gives option to handle in-host logic.
+
+ It is a good option to register in-host event callbacks which are
+ specific for the module. The module is kept in memory for rest of
+ the process.
+
+ Arguments may change in future. E.g. 'host_name' should be possible
+ to receive from 'host' object.
+
+ Args:
+ host (ModuleType): Access to installed/registered host object.
+ host_name (str): Name of host.
+ project_name (str): Project name which is main part of host
+ context.
+ """
+
+ pass
+
def cli(self, module_click_group):
"""Add commands to click group.
@@ -443,6 +506,7 @@ def cli_main():
def mycommand():
print("my_command")
"""
+
pass
@@ -479,6 +543,40 @@ def __init__(self, _system_settings=None):
self.initialize_modules()
self.connect_modules()
+ def __getitem__(self, module_name):
+ return self.modules_by_name[module_name]
+
+ def get(self, module_name, default=None):
+ """Access module by name.
+
+ Args:
+ module_name (str): Name of module which should be returned.
+ default (Any): Default output if module is not available.
+
+ Returns:
+ Union[OpenPypeModule, None]: Module found by name or None.
+ """
+ return self.modules_by_name.get(module_name, default)
+
+ def get_enabled_module(self, module_name, default=None):
+ """Fast access to enabled module.
+
+ If module is available but is not enabled default value is returned.
+
+ Args:
+ module_name (str): Name of module which should be returned.
+ default (Any): Default output if module is not available or is
+ not enabled.
+
+ Returns:
+ Union[OpenPypeModule, None]: Enabled module found by name or None.
+ """
+
+ module = self.get(module_name)
+ if module is not None and module.enabled:
+ return module
+ return default
+
def initialize_modules(self):
"""Import and initialize modules."""
# Make sure modules are loaded
@@ -632,8 +730,6 @@ def collect_plugin_paths(self):
and "actions" each containing list of paths.
"""
# Output structure
- from openpype_interfaces import IPluginPaths
-
output = {
"publish": [],
"create": [],
@@ -680,41 +776,64 @@ def collect_plugin_paths(self):
).format(expected_keys, " | ".join(msg_items)))
return output
- def collect_launch_hook_paths(self):
- """Helper to collect hooks from modules inherited ILaunchHookPaths.
+ def collect_creator_plugin_paths(self, host_name):
+ """Helper to collect creator plugin paths from modules.
+
+ Args:
+ host_name (str): For which host are creators meants.
Returns:
- list: Paths to launch hook directories.
+ list: List of creator plugin paths.
"""
- from openpype_interfaces import ILaunchHookPaths
-
- str_type = type("")
- expected_types = (list, tuple, set)
-
+ # Output structure
output = []
for module in self.get_enabled_modules():
- # Skip module that do not inherit from `ILaunchHookPaths`
- if not isinstance(module, ILaunchHookPaths):
+ # Skip module that do not inherit from `IPluginPaths`
+ if not isinstance(module, IPluginPaths):
continue
- hook_paths = module.get_launch_hook_paths()
- if not hook_paths:
- continue
+ paths = module.get_creator_plugin_paths(host_name)
+ if paths:
+ # Convert to list if value is not list
+ if not isinstance(paths, (list, tuple, set)):
+ paths = [paths]
+ output.extend(paths)
+ return output
- # Convert string to list
- if isinstance(hook_paths, str_type):
- hook_paths = [hook_paths]
+ def get_host_module(self, host_name):
+ """Find host module by host name.
- # Skip invalid types
- if not isinstance(hook_paths, expected_types):
- self.log.warning((
- "Result of `get_launch_hook_paths`"
- " has invalid type {}. Expected {}"
- ).format(type(hook_paths), expected_types))
- continue
+ Args:
+ host_name (str): Host name for which is found host module.
- output.extend(hook_paths)
- return output
+ Returns:
+ OpenPypeModule: Found host module by name.
+ None: There was not found module inheriting IHostAddon which has
+ host name set to passed 'host_name'.
+ """
+
+ for module in self.get_enabled_modules():
+ if (
+ isinstance(module, IHostAddon)
+ and module.host_name == host_name
+ ):
+ return module
+ return None
+
+ def get_host_names(self):
+ """List of available host names based on host modules.
+
+ Returns:
+ Iterable[str]: All available host names based on enabled modules
+ inheriting 'IHostAddon'.
+ """
+
+ host_names = {
+ module.host_name
+ for module in self.get_enabled_modules()
+ if isinstance(module, IHostAddon)
+ }
+ return host_names
def print_report(self):
"""Print out report of time spent on modules initialization parts.
@@ -838,18 +957,20 @@ class TrayModulesManager(ModulesManager):
modules_menu_order = (
"user",
"ftrack",
+ "kitsu",
"muster",
"launcher_tool",
"avalon",
"clockify",
"standalonepublish_tool",
+ "traypublish_tool",
"log_viewer",
"local_settings",
"settings"
)
def __init__(self):
- self.log = PypeLogger.get_logger(self.__class__.__name__)
+ self.log = Logger.get_logger(self.__class__.__name__)
self.modules = []
self.modules_by_id = {}
@@ -888,8 +1009,6 @@ def initialize(self, tray_manager, tray_menu):
self.tray_menu(tray_menu)
def get_enabled_tray_modules(self):
- from openpype_interfaces import ITrayModule
-
output = []
for module in self.modules:
if module.enabled and isinstance(module, ITrayModule):
@@ -965,8 +1084,6 @@ def tray_menu(self, tray_menu):
self._report["Tray menu"] = report
def start_modules(self):
- from openpype_interfaces import ITrayService
-
report = {}
time_start = time.time()
prev_start_time = time_start
@@ -1025,7 +1142,7 @@ def get_module_settings_defs():
settings_defs = []
- log = PypeLogger.get_logger("ModuleSettingsLoad")
+ log = Logger.get_logger("ModuleSettingsLoad")
for raw_module in openpype_modules:
for attr_name in dir(raw_module):
diff --git a/openpype/modules/clockify/launcher_actions/ClockifyStart.py b/openpype/modules/clockify/launcher_actions/ClockifyStart.py
index db51964eb73..7663aecc316 100644
--- a/openpype/modules/clockify/launcher_actions/ClockifyStart.py
+++ b/openpype/modules/clockify/launcher_actions/ClockifyStart.py
@@ -1,13 +1,9 @@
-from avalon import api, io
-from openpype.api import Logger
+from openpype.client import get_asset_by_name
+from openpype.pipeline import LauncherAction
from openpype_modules.clockify.clockify_api import ClockifyAPI
-log = Logger().get_logger(__name__)
-
-
-class ClockifyStart(api.Action):
-
+class ClockifyStart(LauncherAction):
name = "clockify_start_timer"
label = "Clockify - Start Timer"
icon = "clockify_icon"
@@ -21,20 +17,19 @@ def is_compatible(self, session):
return False
def process(self, session, **kwargs):
- project_name = session['AVALON_PROJECT']
- asset_name = session['AVALON_ASSET']
- task_name = session['AVALON_TASK']
+ project_name = session["AVALON_PROJECT"]
+ asset_name = session["AVALON_ASSET"]
+ task_name = session["AVALON_TASK"]
description = asset_name
- asset = io.find_one({
- 'type': 'asset',
- 'name': asset_name
- })
- if asset is not None:
- desc_items = asset.get('data', {}).get('parents', [])
+ asset_doc = get_asset_by_name(
+ project_name, asset_name, fields=["data.parents"]
+ )
+ if asset_doc is not None:
+ desc_items = asset_doc.get("data", {}).get("parents", [])
desc_items.append(asset_name)
desc_items.append(task_name)
- description = '/'.join(desc_items)
+ description = "/".join(desc_items)
project_id = self.clockapi.get_project_id(project_name)
tag_ids = []
diff --git a/openpype/modules/clockify/launcher_actions/ClockifySync.py b/openpype/modules/clockify/launcher_actions/ClockifySync.py
index 02982d373a2..c346a1b4f62 100644
--- a/openpype/modules/clockify/launcher_actions/ClockifySync.py
+++ b/openpype/modules/clockify/launcher_actions/ClockifySync.py
@@ -1,10 +1,9 @@
-from avalon import api, io
+from openpype.client import get_projects, get_project
from openpype_modules.clockify.clockify_api import ClockifyAPI
-from openpype.api import Logger
-log = Logger().get_logger(__name__)
+from openpype.pipeline import LauncherAction
-class ClockifySync(api.Action):
+class ClockifySync(LauncherAction):
name = "sync_to_clockify"
label = "Sync to Clockify"
@@ -18,39 +17,36 @@ def is_compatible(self, session):
return self.have_permissions
def process(self, session, **kwargs):
- project_name = session.get('AVALON_PROJECT', None)
+ project_name = session.get("AVALON_PROJECT") or ""
projects_to_sync = []
- if project_name.strip() == '' or project_name is None:
- for project in io.projects():
- projects_to_sync.append(project)
+ if project_name.strip():
+ projects_to_sync = [get_project(project_name)]
else:
- project = io.find_one({'type': 'project'})
- projects_to_sync.append(project)
+ projects_to_sync = get_projects()
projects_info = {}
for project in projects_to_sync:
- task_types = project['config']['tasks'].keys()
- projects_info[project['name']] = task_types
+ task_types = project["config"]["tasks"].keys()
+ projects_info[project["name"]] = task_types
clockify_projects = self.clockapi.get_projects()
for project_name, task_types in projects_info.items():
- if project_name not in clockify_projects:
- response = self.clockapi.add_project(project_name)
- if 'id' not in response:
- self.log.error('Project {} can\'t be created'.format(
- project_name
- ))
- continue
- project_id = response['id']
- else:
- project_id = clockify_projects[project_name]
+ if project_name in clockify_projects:
+ continue
+
+ response = self.clockapi.add_project(project_name)
+ if "id" not in response:
+ self.log.error("Project {} can't be created".format(
+ project_name
+ ))
+ continue
clockify_workspace_tags = self.clockapi.get_tags()
for task_type in task_types:
if task_type not in clockify_workspace_tags:
response = self.clockapi.add_tag(task_type)
- if 'id' not in response:
+ if "id" not in response:
self.log.error('Task {} can\'t be created'.format(
task_type
))
diff --git a/openpype/modules/default_modules/deadline/__init__.py b/openpype/modules/deadline/__init__.py
similarity index 100%
rename from openpype/modules/default_modules/deadline/__init__.py
rename to openpype/modules/deadline/__init__.py
diff --git a/openpype/modules/deadline/abstract_submit_deadline.py b/openpype/modules/deadline/abstract_submit_deadline.py
new file mode 100644
index 00000000000..512ff800ee6
--- /dev/null
+++ b/openpype/modules/deadline/abstract_submit_deadline.py
@@ -0,0 +1,665 @@
+# -*- coding: utf-8 -*-
+"""Abstract package for submitting jobs to Deadline.
+
+It provides Deadline JobInfo data class.
+
+"""
+import json.decoder
+import os
+from abc import abstractmethod
+import platform
+import getpass
+from functools import partial
+from collections import OrderedDict
+
+import six
+import attr
+import requests
+
+import pyblish.api
+from openpype.pipeline.publish import (
+ AbstractMetaInstancePlugin,
+ KnownPublishError
+)
+
+JSONDecodeError = getattr(json.decoder, "JSONDecodeError", ValueError)
+
+
+def requests_post(*args, **kwargs):
+ """Wrap request post method.
+
+ Disabling SSL certificate validation if ``DONT_VERIFY_SSL`` environment
+ variable is found. This is useful when Deadline or Muster server are
+ running with self-signed certificates and their certificate is not
+ added to trusted certificates on client machines.
+
+ Warning:
+ Disabling SSL certificate validation is defeating one line
+ of defense SSL is providing and it is not recommended.
+
+ """
+ if 'verify' not in kwargs:
+ kwargs['verify'] = False if os.getenv("OPENPYPE_DONT_VERIFY_SSL",
+ True) else True # noqa
+ # add 10sec timeout before bailing out
+ kwargs['timeout'] = 10
+ return requests.post(*args, **kwargs)
+
+
+def requests_get(*args, **kwargs):
+ """Wrap request get method.
+
+ Disabling SSL certificate validation if ``DONT_VERIFY_SSL`` environment
+ variable is found. This is useful when Deadline or Muster server are
+ running with self-signed certificates and their certificate is not
+ added to trusted certificates on client machines.
+
+ Warning:
+ Disabling SSL certificate validation is defeating one line
+ of defense SSL is providing and it is not recommended.
+
+ """
+ if 'verify' not in kwargs:
+ kwargs['verify'] = False if os.getenv("OPENPYPE_DONT_VERIFY_SSL",
+ True) else True # noqa
+ # add 10sec timeout before bailing out
+ kwargs['timeout'] = 10
+ return requests.get(*args, **kwargs)
+
+
+class DeadlineKeyValueVar(dict):
+ """
+
+ Serializes dictionary key values as "{key}={value}" like Deadline uses
+ for EnvironmentKeyValue.
+
+ As an example:
+ EnvironmentKeyValue0="A_KEY=VALUE_A"
+ EnvironmentKeyValue1="OTHER_KEY=VALUE_B"
+
+ The keys are serialized in alphabetical order (sorted).
+
+ Example:
+ >>> var = DeadlineKeyValueVar("EnvironmentKeyValue")
+ >>> var["my_var"] = "hello"
+ >>> var["my_other_var"] = "hello2"
+ >>> var.serialize()
+
+
+ """
+ def __init__(self, key):
+ super(DeadlineKeyValueVar, self).__init__()
+ self.__key = key
+
+ def serialize(self):
+ key = self.__key
+
+ # Allow custom location for index in serialized string
+ if "{}" not in key:
+ key = key + "{}"
+
+ return {
+ key.format(index): "{}={}".format(var_key, var_value)
+ for index, (var_key, var_value) in enumerate(sorted(self.items()))
+ }
+
+
+class DeadlineIndexedVar(dict):
+ """
+
+ Allows to set and query values by integer indices:
+ Query: var[1] or var.get(1)
+ Set: var[1] = "my_value"
+ Append: var += "value"
+
+ Note: Iterating the instance is not guarantueed to be the order of the
+ indices. To do so iterate with `sorted()`
+
+ """
+ def __init__(self, key):
+ super(DeadlineIndexedVar, self).__init__()
+ self.__key = key
+
+ def serialize(self):
+ key = self.__key
+
+ # Allow custom location for index in serialized string
+ if "{}" not in key:
+ key = key + "{}"
+
+ return {
+ key.format(index): value for index, value in sorted(self.items())
+ }
+
+ def next_available_index(self):
+ # Add as first unused entry
+ i = 0
+ while i in self.keys():
+ i += 1
+ return i
+
+ def update(self, data):
+ # Force the integer key check
+ for key, value in data.items():
+ self.__setitem__(key, value)
+
+ def __iadd__(self, other):
+ index = self.next_available_index()
+ self[index] = other
+ return self
+
+ def __setitem__(self, key, value):
+ if not isinstance(key, int):
+ raise TypeError("Key must be an integer: {}".format(key))
+
+ if key < 0:
+ raise ValueError("Negative index can't be set: {}".format(key))
+ dict.__setitem__(self, key, value)
+
+
+@attr.s
+class DeadlineJobInfo(object):
+ """Mapping of all Deadline *JobInfo* attributes.
+
+ This contains all JobInfo attributes plus their default values.
+ Those attributes set to `None` shouldn't be posted to Deadline as
+ the only required one is `Plugin`. Their default values used by Deadline
+ are stated in
+ comments.
+
+ ..seealso:
+ https://docs.thinkboxsoftware.com/products/deadline/10.1/1_User%20Manual/manual/manual-submission.html
+
+ """
+
+ # Required
+ # ----------------------------------------------
+ Plugin = attr.ib()
+
+ # General
+ Frames = attr.ib(default=None) # default: 0
+ Name = attr.ib(default="Untitled")
+ Comment = attr.ib(default=None) # default: empty
+ Department = attr.ib(default=None) # default: empty
+ BatchName = attr.ib(default=None) # default: empty
+ UserName = attr.ib(default=getpass.getuser())
+ MachineName = attr.ib(default=platform.node())
+ Pool = attr.ib(default=None) # default: "none"
+ SecondaryPool = attr.ib(default=None)
+ Group = attr.ib(default=None) # default: "none"
+ Priority = attr.ib(default=50)
+ ChunkSize = attr.ib(default=1)
+ ConcurrentTasks = attr.ib(default=1)
+ LimitConcurrentTasksToNumberOfCpus = attr.ib(
+ default=None) # default: "true"
+ OnJobComplete = attr.ib(default="Nothing")
+ SynchronizeAllAuxiliaryFiles = attr.ib(default=None) # default: false
+ ForceReloadPlugin = attr.ib(default=None) # default: false
+ Sequential = attr.ib(default=None) # default: false
+ SuppressEvents = attr.ib(default=None) # default: false
+ Protected = attr.ib(default=None) # default: false
+ InitialStatus = attr.ib(default="Active")
+ NetworkRoot = attr.ib(default=None)
+
+ # Timeouts
+ # ----------------------------------------------
+ MinRenderTimeSeconds = attr.ib(default=None) # Default: 0
+ MinRenderTimeMinutes = attr.ib(default=None) # Default: 0
+ TaskTimeoutSeconds = attr.ib(default=None) # Default: 0
+ TaskTimeoutMinutes = attr.ib(default=None) # Default: 0
+ StartJobTimeoutSeconds = attr.ib(default=None) # Default: 0
+ StartJobTimeoutMinutes = attr.ib(default=None) # Default: 0
+ InitializePluginTimeoutSeconds = attr.ib(default=None) # Default: 0
+ # can be one of
+ OnTaskTimeout = attr.ib(default=None) # Default: Error
+ EnableTimeoutsForScriptTasks = attr.ib(default=None) # Default: false
+ EnableFrameTimeouts = attr.ib(default=None) # Default: false
+ EnableAutoTimeout = attr.ib(default=None) # Default: false
+
+ # Interruptible
+ # ----------------------------------------------
+ Interruptible = attr.ib(default=None) # Default: false
+ InterruptiblePercentage = attr.ib(default=None)
+ RemTimeThreshold = attr.ib(default=None)
+
+ # Notifications
+ # ----------------------------------------------
+ # can be comma separated list of users
+ NotificationTargets = attr.ib(default=None) # Default: blank
+ ClearNotificationTargets = attr.ib(default=None) # Default: false
+ # A comma separated list of additional email addresses
+ NotificationEmails = attr.ib(default=None) # Default: blank
+ OverrideNotificationMethod = attr.ib(default=None) # Default: false
+ EmailNotification = attr.ib(default=None) # Default: false
+ PopupNotification = attr.ib(default=None) # Default: false
+ # String with `[EOL]` used for end of line
+ NotificationNote = attr.ib(default=None) # Default: blank
+
+ # Machine Limit
+ # ----------------------------------------------
+ MachineLimit = attr.ib(default=None) # Default: 0
+ MachineLimitProgress = attr.ib(default=None) # Default: -1.0
+ Whitelist = attr.ib(default=None) # Default: blank
+ Blacklist = attr.ib(default=None) # Default: blank
+
+ # Limits
+ # ----------------------------------------------
+ # comma separated list of limit groups
+ LimitGroups = attr.ib(default=None) # Default: blank
+
+ # Dependencies
+ # ----------------------------------------------
+ # comma separated list of job IDs
+ JobDependencies = attr.ib(default=None) # Default: blank
+ JobDependencyPercentage = attr.ib(default=None) # Default: -1
+ IsFrameDependent = attr.ib(default=None) # Default: false
+ FrameDependencyOffsetStart = attr.ib(default=None) # Default: 0
+ FrameDependencyOffsetEnd = attr.ib(default=None) # Default: 0
+ ResumeOnCompleteDependencies = attr.ib(default=None) # Default: true
+ ResumeOnDeletedDependencies = attr.ib(default=None) # Default: false
+ ResumeOnFailedDependencies = attr.ib(default=None) # Default: false
+ # comma separated list of asset paths
+ RequiredAssets = attr.ib(default=None) # Default: blank
+ # comma separated list of script paths
+ ScriptDependencies = attr.ib(default=None) # Default: blank
+
+ # Failure Detection
+ # ----------------------------------------------
+ OverrideJobFailureDetection = attr.ib(default=None) # Default: false
+ FailureDetectionJobErrors = attr.ib(default=None) # 0..x
+ OverrideTaskFailureDetection = attr.ib(default=None) # Default: false
+ FailureDetectionTaskErrors = attr.ib(default=None) # 0..x
+ IgnoreBadJobDetection = attr.ib(default=None) # Default: false
+ SendJobErrorWarning = attr.ib(default=None) # Default: false
+
+ # Cleanup
+ # ----------------------------------------------
+ DeleteOnComplete = attr.ib(default=None) # Default: false
+ ArchiveOnComplete = attr.ib(default=None) # Default: false
+ OverrideAutoJobCleanup = attr.ib(default=None) # Default: false
+ OverrideJobCleanup = attr.ib(default=None)
+ JobCleanupDays = attr.ib(default=None) # Default: false
+ #
+ OverrideJobCleanupType = attr.ib(default=None)
+
+ # Scheduling
+ # ----------------------------------------------
+ #
+ ScheduledType = attr.ib(default=None) # Default: None
+ #
+ ScheduledStartDateTime = attr.ib(default=None)
+ ScheduledDays = attr.ib(default=None) # Default: 1
+ #
+ JobDelay = attr.ib(default=None)
+ # Time=
+ Scheduled = attr.ib(default=None)
+
+ # Scripts
+ # ----------------------------------------------
+ # all accept path to script
+ PreJobScript = attr.ib(default=None) # Default: blank
+ PostJobScript = attr.ib(default=None) # Default: blank
+ PreTaskScript = attr.ib(default=None) # Default: blank
+ PostTaskScript = attr.ib(default=None) # Default: blank
+
+ # Event Opt-Ins
+ # ----------------------------------------------
+ # comma separated list of plugins
+ EventOptIns = attr.ib(default=None) # Default: blank
+
+ # Environment
+ # ----------------------------------------------
+ EnvironmentKeyValue = attr.ib(factory=partial(DeadlineKeyValueVar,
+ "EnvironmentKeyValue"))
+
+ IncludeEnvironment = attr.ib(default=None) # Default: false
+ UseJobEnvironmentOnly = attr.ib(default=None) # Default: false
+ CustomPluginDirectory = attr.ib(default=None) # Default: blank
+
+ # Job Extra Info
+ # ----------------------------------------------
+ ExtraInfo = attr.ib(factory=partial(DeadlineIndexedVar, "ExtraInfo"))
+ ExtraInfoKeyValue = attr.ib(factory=partial(DeadlineKeyValueVar,
+ "ExtraInfoKeyValue"))
+
+ # Task Extra Info Names
+ # ----------------------------------------------
+ OverrideTaskExtraInfoNames = attr.ib(default=None) # Default: false
+ TaskExtraInfoName = attr.ib(factory=partial(DeadlineIndexedVar,
+ "TaskExtraInfoName"))
+
+ # Output
+ # ----------------------------------------------
+ OutputFilename = attr.ib(factory=partial(DeadlineIndexedVar,
+ "OutputFilename"))
+ OutputFilenameTile = attr.ib(factory=partial(DeadlineIndexedVar,
+ "OutputFilename{}Tile"))
+ OutputDirectory = attr.ib(factory=partial(DeadlineIndexedVar,
+ "OutputDirectory"))
+
+ # Asset Dependency
+ # ----------------------------------------------
+ AssetDependency = attr.ib(factory=partial(DeadlineIndexedVar,
+ "AssetDependency"))
+
+ # Tile Job
+ # ----------------------------------------------
+ TileJob = attr.ib(default=None) # Default: false
+ TileJobFrame = attr.ib(default=None) # Default: 0
+ TileJobTilesInX = attr.ib(default=None) # Default: 0
+ TileJobTilesInY = attr.ib(default=None) # Default: 0
+ TileJobTileCount = attr.ib(default=None) # Default: 0
+
+ # Maintenance Job
+ # ----------------------------------------------
+ MaintenanceJob = attr.ib(default=None) # Default: false
+ MaintenanceJobStartFrame = attr.ib(default=None) # Default: 0
+ MaintenanceJobEndFrame = attr.ib(default=None) # Default: 0
+
+ def serialize(self):
+ """Return all data serialized as dictionary.
+
+ Returns:
+ OrderedDict: all serialized data.
+
+ """
+ def filter_data(a, v):
+ if isinstance(v, (DeadlineIndexedVar, DeadlineKeyValueVar)):
+ return False
+ if v is None:
+ return False
+ return True
+
+ serialized = attr.asdict(
+ self, dict_factory=OrderedDict, filter=filter_data)
+
+ # Custom serialize these attributes
+ for attribute in [
+ self.EnvironmentKeyValue,
+ self.ExtraInfo,
+ self.ExtraInfoKeyValue,
+ self.TaskExtraInfoName,
+ self.OutputFilename,
+ self.OutputFilenameTile,
+ self.OutputDirectory,
+ self.AssetDependency
+ ]:
+ serialized.update(attribute.serialize())
+
+ return serialized
+
+ def update(self, data):
+ """Update instance with data dict"""
+ for key, value in data.items():
+ setattr(self, key, value)
+
+
+@six.add_metaclass(AbstractMetaInstancePlugin)
+class AbstractSubmitDeadline(pyblish.api.InstancePlugin):
+ """Class abstracting access to Deadline."""
+
+ label = "Submit to Deadline"
+ order = pyblish.api.IntegratorOrder + 0.1
+ use_published = True
+ asset_dependencies = False
+
+ def __init__(self, *args, **kwargs):
+ super(AbstractSubmitDeadline, self).__init__(*args, **kwargs)
+ self._instance = None
+ self._deadline_url = None
+ self.scene_path = None
+ self.job_info = None
+ self.plugin_info = None
+ self.aux_files = None
+
+ def process(self, instance):
+ """Plugin entry point."""
+ self._instance = instance
+ context = instance.context
+ self._deadline_url = context.data.get("defaultDeadline")
+ self._deadline_url = instance.data.get(
+ "deadlineUrl", self._deadline_url)
+
+ assert self._deadline_url, "Requires Deadline Webservice URL"
+
+ file_path = None
+ if self.use_published:
+ file_path = self.from_published_scene()
+
+ # fallback if nothing was set
+ if not file_path:
+ self.log.warning("Falling back to workfile")
+ file_path = context.data["currentFile"]
+
+ self.scene_path = file_path
+ self.log.info("Using {} for render/export.".format(file_path))
+
+ self.job_info = self.get_job_info()
+ self.plugin_info = self.get_plugin_info()
+ self.aux_files = self.get_aux_files()
+
+ self.process_submission()
+
+ def process_submission(self):
+ """Process data for submission.
+
+ This takes Deadline JobInfo, PluginInfo, AuxFile, creates payload
+ from them and submit it do Deadline.
+
+ Returns:
+ str: Deadline job ID
+
+ """
+ payload = self.assemble_payload()
+ return self.submit(payload)
+
+ @abstractmethod
+ def get_job_info(self):
+ """Return filled Deadline JobInfo.
+
+ This is host/plugin specific implementation of how to fill data in.
+
+ See:
+ :class:`DeadlineJobInfo`
+
+ Returns:
+ :class:`DeadlineJobInfo`: Filled Deadline JobInfo.
+
+ """
+ pass
+
+ @abstractmethod
+ def get_plugin_info(self):
+ """Return filled Deadline PluginInfo.
+
+ This is host/plugin specific implementation of how to fill data in.
+
+ See:
+ :class:`DeadlineJobInfo`
+
+ Returns:
+ dict: Filled Deadline JobInfo.
+
+ """
+ pass
+
+ def get_aux_files(self):
+ """Return list of auxiliary files for Deadline job.
+
+ If needed this should be overridden, otherwise return empty list as
+ that field even empty must be present on Deadline submission.
+
+ Returns:
+ list: List of files.
+
+ """
+ return []
+
+ def from_published_scene(self, replace_in_path=True):
+ """Switch work scene for published scene.
+
+ If rendering/exporting from published scenes is enabled, this will
+ replace paths from working scene to published scene.
+
+ Args:
+ replace_in_path (bool): if True, it will try to find
+ old scene name in path of expected files and replace it
+ with name of published scene.
+
+ Returns:
+ str: Published scene path.
+ None: if no published scene is found.
+
+ Note:
+ Published scene path is actually determined from project Anatomy
+ as at the time this plugin is running scene can still no be
+ published.
+
+ """
+
+ instance = self._instance
+ workfile_instance = self._get_workfile_instance(instance.context)
+ if workfile_instance is None:
+ return
+
+ # determine published path from Anatomy.
+ template_data = workfile_instance.data.get("anatomyData")
+ rep = workfile_instance.data.get("representations")[0]
+ template_data["representation"] = rep.get("name")
+ template_data["ext"] = rep.get("ext")
+ template_data["comment"] = None
+
+ anatomy = instance.context.data['anatomy']
+ anatomy_filled = anatomy.format(template_data)
+ template_filled = anatomy_filled["publish"]["path"]
+ file_path = os.path.normpath(template_filled)
+
+ self.log.info("Using published scene for render {}".format(file_path))
+
+ if not os.path.exists(file_path):
+ self.log.error("published scene does not exist!")
+ raise
+
+ if not replace_in_path:
+ return file_path
+
+ # now we need to switch scene in expected files
+ # because token will now point to published
+ # scene file and that might differ from current one
+ def _clean_name(path):
+ return os.path.splitext(os.path.basename(path))[0]
+
+ new_scene = _clean_name(file_path)
+ orig_scene = _clean_name(instance.context.data["currentFile"])
+ expected_files = instance.data.get("expectedFiles")
+
+ if isinstance(expected_files[0], dict):
+ # we have aovs and we need to iterate over them
+ new_exp = {}
+ for aov, files in expected_files[0].items():
+ replaced_files = []
+ for f in files:
+ replaced_files.append(
+ str(f).replace(orig_scene, new_scene)
+ )
+ new_exp[aov] = replaced_files
+ # [] might be too much here, TODO
+ instance.data["expectedFiles"] = [new_exp]
+ else:
+ new_exp = []
+ for f in expected_files:
+ new_exp.append(
+ str(f).replace(orig_scene, new_scene)
+ )
+ instance.data["expectedFiles"] = new_exp
+
+ metadata_folder = instance.data.get("publishRenderMetadataFolder")
+ if metadata_folder:
+ metadata_folder = metadata_folder.replace(orig_scene,
+ new_scene)
+ instance.data["publishRenderMetadataFolder"] = metadata_folder
+
+ self.log.info("Scene name was switched {} -> {}".format(
+ orig_scene, new_scene
+ ))
+
+ return file_path
+
+ def assemble_payload(
+ self, job_info=None, plugin_info=None, aux_files=None):
+ """Assemble payload data from its various parts.
+
+ Args:
+ job_info (DeadlineJobInfo): Deadline JobInfo. You can use
+ :class:`DeadlineJobInfo` for it.
+ plugin_info (dict): Deadline PluginInfo. Plugin specific options.
+ aux_files (list, optional): List of auxiliary file to submit with
+ the job.
+
+ Returns:
+ dict: Deadline Payload.
+
+ """
+ job = job_info or self.job_info
+ return {
+ "JobInfo": job.serialize(),
+ "PluginInfo": plugin_info or self.plugin_info,
+ "AuxFiles": aux_files or self.aux_files
+ }
+
+ def submit(self, payload):
+ """Submit payload to Deadline API end-point.
+
+ This takes payload in the form of JSON file and POST it to
+ Deadline jobs end-point.
+
+ Args:
+ payload (dict): dict to become json in deadline submission.
+
+ Returns:
+ str: resulting Deadline job id.
+
+ Throws:
+ KnownPublishError: if submission fails.
+
+ """
+ url = "{}/api/jobs".format(self._deadline_url)
+ response = requests_post(url, json=payload)
+ if not response.ok:
+ self.log.error("Submission failed!")
+ self.log.error(response.status_code)
+ self.log.error(response.content)
+ self.log.debug(payload)
+ raise KnownPublishError(response.text)
+
+ try:
+ result = response.json()
+ except JSONDecodeError:
+ msg = "Broken response {}. ".format(response)
+ msg += "Try restarting the Deadline Webservice."
+ self.log.warning(msg, exc_info=True)
+ raise KnownPublishError("Broken response from DL")
+
+ # for submit publish job
+ self._instance.data["deadlineSubmissionJob"] = result
+
+ return result["_id"]
+
+ @staticmethod
+ def _get_workfile_instance(context):
+ """Find workfile instance in context"""
+ for i in context:
+
+ is_workfile = (
+ "workfile" in i.data.get("families", []) or
+ i.data["family"] == "workfile"
+ )
+ if not is_workfile:
+ continue
+
+ # test if there is instance of workfile waiting
+ # to be published.
+ assert i.data["publish"] is True, (
+ "Workfile (scene) must be published along")
+
+ return i
diff --git a/openpype/modules/deadline/deadline_module.py b/openpype/modules/deadline/deadline_module.py
new file mode 100644
index 00000000000..bbd0f74e8a9
--- /dev/null
+++ b/openpype/modules/deadline/deadline_module.py
@@ -0,0 +1,77 @@
+import os
+import requests
+import six
+import sys
+
+from openpype.lib import requests_get, Logger
+from openpype.modules import OpenPypeModule
+from openpype_interfaces import IPluginPaths
+
+
+class DeadlineWebserviceError(Exception):
+ """
+ Exception to throw when connection to Deadline server fails.
+ """
+
+
+class DeadlineModule(OpenPypeModule, IPluginPaths):
+ name = "deadline"
+
+ def __init__(self, manager, settings):
+ self.deadline_urls = {}
+ super(DeadlineModule, self).__init__(manager, settings)
+
+ def initialize(self, modules_settings):
+ # This module is always enabled
+ deadline_settings = modules_settings[self.name]
+ self.enabled = deadline_settings["enabled"]
+ deadline_url = deadline_settings.get("DEADLINE_REST_URL")
+ if deadline_url:
+ self.deadline_urls = {"default": deadline_url}
+ else:
+ self.deadline_urls = deadline_settings.get("deadline_urls") # noqa: E501
+
+ if not self.deadline_urls:
+ self.enabled = False
+ self.log.warning(("default Deadline Webservice URL "
+ "not specified. Disabling module."))
+ return
+
+ def get_plugin_paths(self):
+ """Deadline plugin paths."""
+ current_dir = os.path.dirname(os.path.abspath(__file__))
+ return {
+ "publish": [os.path.join(current_dir, "plugins", "publish")]
+ }
+
+ @staticmethod
+ def get_deadline_pools(webservice, log=None):
+ # type: (str) -> list
+ """Get pools from Deadline.
+ Args:
+ webservice (str): Server url.
+ log (Logger)
+ Returns:
+ list: Pools.
+ Throws:
+ RuntimeError: If deadline webservice is unreachable.
+
+ """
+ if not log:
+ log = Logger.get_logger(__name__)
+
+ argument = "{}/api/pools?NamesOnly=true".format(webservice)
+ try:
+ response = requests_get(argument)
+ except requests.exceptions.ConnectionError as exc:
+ msg = 'Cannot connect to DL web service {}'.format(webservice)
+ log.error(msg)
+ six.reraise(
+ DeadlineWebserviceError,
+ DeadlineWebserviceError('{} - {}'.format(msg, exc)),
+ sys.exc_info()[2])
+ if not response.ok:
+ log.warning("No pools retrieved")
+ return []
+
+ return response.json()
diff --git a/openpype/modules/default_modules/deadline/plugins/publish/collect_deadline_server_from_instance.py b/openpype/modules/deadline/plugins/publish/collect_deadline_server_from_instance.py
similarity index 96%
rename from openpype/modules/default_modules/deadline/plugins/publish/collect_deadline_server_from_instance.py
rename to openpype/modules/deadline/plugins/publish/collect_deadline_server_from_instance.py
index 1bc4eaa0675..9981bead3e3 100644
--- a/openpype/modules/default_modules/deadline/plugins/publish/collect_deadline_server_from_instance.py
+++ b/openpype/modules/deadline/plugins/publish/collect_deadline_server_from_instance.py
@@ -11,9 +11,9 @@
class CollectDeadlineServerFromInstance(pyblish.api.InstancePlugin):
"""Collect Deadline Webservice URL from instance."""
- order = pyblish.api.CollectorOrder + 0.02
+ order = pyblish.api.CollectorOrder + 0.415
label = "Deadline Webservice from the Instance"
- families = ["rendering"]
+ families = ["rendering", "renderlayer"]
def process(self, instance):
instance.data["deadlineUrl"] = self._collect_deadline_url(instance)
diff --git a/openpype/modules/default_modules/deadline/plugins/publish/collect_default_deadline_server.py b/openpype/modules/deadline/plugins/publish/collect_default_deadline_server.py
similarity index 84%
rename from openpype/modules/default_modules/deadline/plugins/publish/collect_default_deadline_server.py
rename to openpype/modules/deadline/plugins/publish/collect_default_deadline_server.py
index 53231bd7e42..e6ad6a9aa1b 100644
--- a/openpype/modules/default_modules/deadline/plugins/publish/collect_default_deadline_server.py
+++ b/openpype/modules/deadline/plugins/publish/collect_default_deadline_server.py
@@ -6,9 +6,11 @@
class CollectDefaultDeadlineServer(pyblish.api.ContextPlugin):
"""Collect default Deadline Webservice URL."""
- order = pyblish.api.CollectorOrder + 0.01
+ order = pyblish.api.CollectorOrder + 0.410
label = "Default Deadline Webservice"
+ pass_mongo_url = False
+
def process(self, context):
try:
deadline_module = context.data.get("openPypeModules")["deadline"]
@@ -19,3 +21,5 @@ def process(self, context):
# get default deadline webservice url from deadline module
self.log.debug(deadline_module.deadline_urls)
context.data["defaultDeadline"] = deadline_module.deadline_urls["default"] # noqa: E501
+
+ context.data["deadlinePassMongoUrl"] = self.pass_mongo_url
diff --git a/openpype/modules/deadline/plugins/publish/collect_pools.py b/openpype/modules/deadline/plugins/publish/collect_pools.py
new file mode 100644
index 00000000000..48130848d51
--- /dev/null
+++ b/openpype/modules/deadline/plugins/publish/collect_pools.py
@@ -0,0 +1,23 @@
+# -*- coding: utf-8 -*-
+"""Collect Deadline pools. Choose default one from Settings
+
+"""
+import pyblish.api
+
+
+class CollectDeadlinePools(pyblish.api.InstancePlugin):
+ """Collect pools from instance if present, from Setting otherwise."""
+
+ order = pyblish.api.CollectorOrder + 0.420
+ label = "Collect Deadline Pools"
+ families = ["rendering", "render.farm", "renderFarm", "renderlayer"]
+
+ primary_pool = None
+ secondary_pool = None
+
+ def process(self, instance):
+ if not instance.data.get("primaryPool"):
+ instance.data["primaryPool"] = self.primary_pool or "none"
+
+ if not instance.data.get("secondaryPool"):
+ instance.data["secondaryPool"] = self.secondary_pool or "none"
diff --git a/openpype/modules/deadline/plugins/publish/collect_publishable_instances.py b/openpype/modules/deadline/plugins/publish/collect_publishable_instances.py
new file mode 100644
index 00000000000..b00381b6cf7
--- /dev/null
+++ b/openpype/modules/deadline/plugins/publish/collect_publishable_instances.py
@@ -0,0 +1,39 @@
+# -*- coding: utf-8 -*-
+"""Collect instances that should be processed and published on DL.
+
+"""
+import os
+
+import pyblish.api
+from openpype.pipeline import PublishValidationError
+
+
+class CollectDeadlinePublishableInstances(pyblish.api.InstancePlugin):
+ """Collect instances that should be processed and published on DL.
+
+ Some long running publishes (not just renders) could be offloaded to DL,
+ this plugin compares theirs name against env variable, marks only
+ publishable by farm.
+
+ Triggered only when running only in headless mode, eg on a farm.
+ """
+
+ order = pyblish.api.CollectorOrder + 0.499
+ label = "Collect Deadline Publishable Instance"
+ targets = ["remote"]
+
+ def process(self, instance):
+ self.log.debug("CollectDeadlinePublishableInstances")
+ publish_inst = os.environ.get("OPENPYPE_PUBLISH_SUBSET", '')
+ if not publish_inst:
+ raise PublishValidationError("OPENPYPE_PUBLISH_SUBSET env var "
+ "required for remote publishing")
+
+ subset_name = instance.data["subset"]
+ if subset_name == publish_inst:
+ self.log.debug("Publish {}".format(subset_name))
+ instance.data["publish"] = True
+ instance.data["farm"] = False
+ else:
+ self.log.debug("Skipping {}".format(subset_name))
+ instance.data["publish"] = False
diff --git a/openpype/modules/deadline/plugins/publish/help/validate_deadline_pools.xml b/openpype/modules/deadline/plugins/publish/help/validate_deadline_pools.xml
new file mode 100644
index 00000000000..0e7d72910e9
--- /dev/null
+++ b/openpype/modules/deadline/plugins/publish/help/validate_deadline_pools.xml
@@ -0,0 +1,31 @@
+
+
+
+ Scene setting
+
+ ## Invalid Deadline pools found
+
+ Configured pools don't match what is set in Deadline.
+
+ {invalid_value_str}
+
+ ### How to repair?
+
+ If your instance had deadline pools set on creation, remove or
+ change them.
+
+ In other cases inform admin to change them in Settings.
+
+ Available deadline pools {pools_str}.
+
+
+ ### __Detailed Info__
+
+ This error is shown when deadline pool is not on Deadline anymore. It
+ could happen in case of republish old workfile which was created with
+ previous deadline pools,
+ or someone changed pools on Deadline side, but didn't modify Openpype
+ Settings.
+
+
+
\ No newline at end of file
diff --git a/openpype/modules/deadline/plugins/publish/submit_aftereffects_deadline.py b/openpype/modules/deadline/plugins/publish/submit_aftereffects_deadline.py
new file mode 100644
index 00000000000..0c1ffa6bd74
--- /dev/null
+++ b/openpype/modules/deadline/plugins/publish/submit_aftereffects_deadline.py
@@ -0,0 +1,139 @@
+import os
+import attr
+import getpass
+import pyblish.api
+
+from openpype.lib import (
+ env_value_to_bool,
+ collect_frames,
+)
+from openpype.pipeline import legacy_io
+from openpype_modules.deadline import abstract_submit_deadline
+from openpype_modules.deadline.abstract_submit_deadline import DeadlineJobInfo
+
+
+@attr.s
+class DeadlinePluginInfo():
+ Comp = attr.ib(default=None)
+ SceneFile = attr.ib(default=None)
+ OutputFilePath = attr.ib(default=None)
+ Output = attr.ib(default=None)
+ StartupDirectory = attr.ib(default=None)
+ Arguments = attr.ib(default=None)
+ ProjectPath = attr.ib(default=None)
+ AWSAssetFile0 = attr.ib(default=None)
+ Version = attr.ib(default=None)
+ MultiProcess = attr.ib(default=None)
+
+
+class AfterEffectsSubmitDeadline(
+ abstract_submit_deadline.AbstractSubmitDeadline
+):
+
+ label = "Submit AE to Deadline"
+ order = pyblish.api.IntegratorOrder + 0.1
+ hosts = ["aftereffects"]
+ families = ["render.farm"] # cannot be "render' as that is integrated
+ use_published = True
+ targets = ["local"]
+
+ priority = 50
+ chunk_size = 1000000
+ group = None
+ department = None
+ multiprocess = True
+
+ def get_job_info(self):
+ dln_job_info = DeadlineJobInfo(Plugin="AfterEffects")
+
+ context = self._instance.context
+
+ dln_job_info.Name = self._instance.data["name"]
+ dln_job_info.BatchName = os.path.basename(self._instance.
+ data["source"])
+ dln_job_info.Plugin = "AfterEffects"
+ dln_job_info.UserName = context.data.get(
+ "deadlineUser", getpass.getuser())
+ if self._instance.data["frameEnd"] > self._instance.data["frameStart"]:
+ # Deadline requires integers in frame range
+ frame_range = "{}-{}".format(
+ int(round(self._instance.data["frameStart"])),
+ int(round(self._instance.data["frameEnd"])))
+ dln_job_info.Frames = frame_range
+
+ dln_job_info.Priority = self.priority
+ dln_job_info.Pool = self._instance.data.get("primaryPool")
+ dln_job_info.SecondaryPool = self._instance.data.get("secondaryPool")
+ dln_job_info.Group = self.group
+ dln_job_info.Department = self.department
+ dln_job_info.ChunkSize = self.chunk_size
+ dln_job_info.OutputFilename += \
+ os.path.basename(self._instance.data["expectedFiles"][0])
+ dln_job_info.OutputDirectory += \
+ os.path.dirname(self._instance.data["expectedFiles"][0])
+ dln_job_info.JobDelay = "00:00:00"
+
+ keys = [
+ "FTRACK_API_KEY",
+ "FTRACK_API_USER",
+ "FTRACK_SERVER",
+ "AVALON_PROJECT",
+ "AVALON_ASSET",
+ "AVALON_TASK",
+ "AVALON_APP_NAME",
+ "OPENPYPE_DEV",
+ "OPENPYPE_LOG_NO_COLORS",
+ "OPENPYPE_VERSION"
+ ]
+ # Add mongo url if it's enabled
+ if self._instance.context.data.get("deadlinePassMongoUrl"):
+ keys.append("OPENPYPE_MONGO")
+
+ environment = dict({key: os.environ[key] for key in keys
+ if key in os.environ}, **legacy_io.Session)
+ for key in keys:
+ value = environment.get(key)
+ if value:
+ dln_job_info.EnvironmentKeyValue[key] = value
+
+ # to recognize job from PYPE for turning Event On/Off
+ dln_job_info.EnvironmentKeyValue["OPENPYPE_RENDER_JOB"] = "1"
+
+ return dln_job_info
+
+ def get_plugin_info(self):
+ deadline_plugin_info = DeadlinePluginInfo()
+
+ render_path = self._instance.data["expectedFiles"][0]
+
+ file_name, frame = list(collect_frames([render_path]).items())[0]
+ if frame:
+ # replace frame ('000001') with Deadline's required '[#######]'
+ # expects filename in format project_asset_subset_version.FRAME.ext
+ render_dir = os.path.dirname(render_path)
+ file_name = os.path.basename(render_path)
+ hashed = '[{}]'.format(len(frame) * "#")
+ file_name = file_name.replace(frame, hashed)
+ render_path = os.path.join(render_dir, file_name)
+
+ deadline_plugin_info.Comp = self._instance.data["comp_name"]
+ deadline_plugin_info.Version = self._instance.data["app_version"]
+ # must be here because of DL AE plugin
+ # added override of multiprocess by env var, if shouldn't be used for
+ # some app variant use MULTIPROCESS:false in Settings, default is True
+ env_multi = env_value_to_bool("MULTIPROCESS", default=True)
+ deadline_plugin_info.MultiProcess = env_multi and self.multiprocess
+ deadline_plugin_info.SceneFile = self.scene_path
+ deadline_plugin_info.Output = render_path.replace("\\", "/")
+
+ return attr.asdict(deadline_plugin_info)
+
+ def from_published_scene(self):
+ """ Do not overwrite expected files.
+
+ Use published is set to True, so rendering will be triggered
+ from published scene (in 'publish' folder). Default implementation
+ of abstract class renames expected (eg. rendered) files accordingly
+ which is not needed here.
+ """
+ return super().from_published_scene(False)
diff --git a/openpype/modules/default_modules/deadline/plugins/publish/submit_harmony_deadline.py b/openpype/modules/deadline/plugins/publish/submit_harmony_deadline.py
similarity index 90%
rename from openpype/modules/default_modules/deadline/plugins/publish/submit_harmony_deadline.py
rename to openpype/modules/deadline/plugins/publish/submit_harmony_deadline.py
index 37041a84b10..6327143623e 100644
--- a/openpype/modules/default_modules/deadline/plugins/publish/submit_harmony_deadline.py
+++ b/openpype/modules/deadline/plugins/publish/submit_harmony_deadline.py
@@ -9,9 +9,9 @@
import attr
import pyblish.api
-import openpype.lib.abstract_submit_deadline
-from openpype.lib.abstract_submit_deadline import DeadlineJobInfo
-from avalon import api
+from openpype.pipeline import legacy_io
+from openpype_modules.deadline import abstract_submit_deadline
+from openpype_modules.deadline.abstract_submit_deadline import DeadlineJobInfo
class _ZipFile(ZipFile):
@@ -217,7 +217,8 @@ def filter_data(a, v):
class HarmonySubmitDeadline(
- openpype.lib.abstract_submit_deadline.AbstractSubmitDeadline):
+ abstract_submit_deadline.AbstractSubmitDeadline
+):
"""Submit render write of Harmony scene to Deadline.
Renders are submitted to a Deadline Web Service as
@@ -237,17 +238,14 @@ class HarmonySubmitDeadline(
order = pyblish.api.IntegratorOrder + 0.1
hosts = ["harmony"]
families = ["render.farm"]
- if not os.environ.get("DEADLINE_REST_URL"):
- optional = False
- active = False
- else:
- optional = True
+ targets = ["local"]
+ optional = True
use_published = False
- primary_pool = ""
- secondary_pool = ""
priority = 50
chunk_size = 1000000
+ group = "none"
+ department = ""
def get_job_info(self):
job_info = DeadlineJobInfo("Harmony")
@@ -260,10 +258,12 @@ def get_job_info(self):
# for now, get those from presets. Later on it should be
# configurable in Harmony UI directly.
job_info.Priority = self.priority
- job_info.Pool = self.primary_pool
- job_info.SecondaryPool = self.secondary_pool
+ job_info.Pool = self._instance.data.get("primaryPool")
+ job_info.SecondaryPool = self._instance.data.get("secondaryPool")
job_info.ChunkSize = self.chunk_size
job_info.BatchName = os.path.basename(self._instance.data["source"])
+ job_info.Department = self.department
+ job_info.Group = self.group
keys = [
"FTRACK_API_KEY",
@@ -274,20 +274,22 @@ def get_job_info(self):
"AVALON_TASK",
"AVALON_APP_NAME",
"OPENPYPE_DEV",
- "OPENPYPE_LOG_NO_COLORS"
+ "OPENPYPE_LOG_NO_COLORS",
+ "OPENPYPE_VERSION"
]
+ # Add mongo url if it's enabled
+ if self._instance.context.data.get("deadlinePassMongoUrl"):
+ keys.append("OPENPYPE_MONGO")
environment = dict({key: os.environ[key] for key in keys
- if key in os.environ}, **api.Session)
+ if key in os.environ}, **legacy_io.Session)
for key in keys:
- val = environment.get(key)
- if val:
- job_info.EnvironmentKeyValue = "{key}={value}".format(
- key=key,
- value=val)
+ value = environment.get(key)
+ if value:
+ job_info.EnvironmentKeyValue[key] = value
# to recognize job from PYPE for turning Event On/Off
- job_info.EnvironmentKeyValue = "OPENPYPE_RENDER_JOB=1"
+ job_info.EnvironmentKeyValue["OPENPYPE_RENDER_JOB"] = "1"
return job_info
@@ -317,10 +319,11 @@ def _unzip_scene_file(self, published_scene: Path) -> Path:
/ published_scene.stem
/ f"{published_scene.stem}.xstage"
)
-
unzip_dir = (published_scene.parent / published_scene.stem)
with _ZipFile(published_scene, "r") as zip_ref:
- zip_ref.extractall(unzip_dir.as_posix())
+ # UNC path (//?/) added to minimalize risk with extracting
+ # to large file paths
+ zip_ref.extractall("//?/" + str(unzip_dir.as_posix()))
# find any xstage files in directory, prefer the one with the same name
# as directory (plus extension)
@@ -348,12 +351,9 @@ def _unzip_scene_file(self, published_scene: Path) -> Path:
# use that one.
if not ideal_scene:
xstage_path = xstage_files[0]
-
return xstage_path
def get_plugin_info(self):
- work_scene = Path(self._instance.data["source"])
-
# this is path to published scene workfile _ZIP_. Before
# rendering, we need to unzip it.
published_scene = Path(
@@ -365,14 +365,13 @@ def get_plugin_info(self):
# for submit_publish job to create .json file in
self._instance.data["outputDir"] = render_path
new_expected_files = []
- work_path_str = str(work_scene.parent.as_posix())
render_path_str = str(render_path.as_posix())
for file in self._instance.data["expectedFiles"]:
_file = str(Path(file).as_posix())
+ expected_dir_str = os.path.dirname(_file)
new_expected_files.append(
- _file.replace(work_path_str, render_path_str)
+ _file.replace(expected_dir_str, render_path_str)
)
-
audio_file = self._instance.data.get("audioFile")
if audio_file:
abs_path = xstage_path.parent / audio_file
diff --git a/openpype/modules/default_modules/deadline/plugins/publish/submit_houdini_remote_publish.py b/openpype/modules/deadline/plugins/publish/submit_houdini_remote_publish.py
similarity index 88%
rename from openpype/modules/default_modules/deadline/plugins/publish/submit_houdini_remote_publish.py
rename to openpype/modules/deadline/plugins/publish/submit_houdini_remote_publish.py
index c3228bfe523..95856137e21 100644
--- a/openpype/modules/default_modules/deadline/plugins/publish/submit_houdini_remote_publish.py
+++ b/openpype/modules/deadline/plugins/publish/submit_houdini_remote_publish.py
@@ -4,10 +4,10 @@
import requests
import hou
-from avalon import api, io
-
import pyblish.api
+from openpype.pipeline import legacy_io
+
class HoudiniSubmitPublishDeadline(pyblish.api.ContextPlugin):
"""Submit Houdini scene to perform a local publish in Deadline.
@@ -35,7 +35,7 @@ def process(self, context):
), "Errors found, aborting integration.."
# Deadline connection
- AVALON_DEADLINE = api.Session.get(
+ AVALON_DEADLINE = legacy_io.Session.get(
"AVALON_DEADLINE", "http://localhost:8082"
)
assert AVALON_DEADLINE, "Requires AVALON_DEADLINE"
@@ -55,7 +55,7 @@ def process(self, context):
scenename = os.path.basename(scene)
# Get project code
- project = io.find_one({"type": "project"})
+ project = context.data["projectEntity"]
code = project["data"].get("code", project["name"])
job_name = "{scene} [PUBLISH]".format(scene=scenename)
@@ -105,15 +105,21 @@ def process(self, context):
# Clarify job name per submission (include instance name)
payload["JobInfo"]["Name"] = job_name + " - %s" % instance
self.submit_job(
- payload, instances=[instance], deadline=AVALON_DEADLINE
+ context,
+ payload,
+ instances=[instance],
+ deadline=AVALON_DEADLINE
)
else:
# Submit a single job
self.submit_job(
- payload, instances=instance_names, deadline=AVALON_DEADLINE
+ context,
+ payload,
+ instances=instance_names,
+ deadline=AVALON_DEADLINE
)
- def submit_job(self, payload, instances, deadline):
+ def submit_job(self, context, payload, instances, deadline):
# Ensure we operate on a copy, a shallow copy is fine.
payload = payload.copy()
@@ -124,11 +130,15 @@ def submit_job(self, payload, instances, deadline):
# this application with so the Render Slave can build its own
# similar environment using it, e.g. "houdini17.5;pluginx2.3"
"AVALON_TOOLS",
+ "OPENPYPE_VERSION"
]
+ # Add mongo url if it's enabled
+ if context.data.get("deadlinePassMongoUrl"):
+ keys.append("OPENPYPE_MONGO")
environment = dict(
{key: os.environ[key] for key in keys if key in os.environ},
- **api.Session
+ **legacy_io.Session
)
environment["PYBLISH_ACTIVE_INSTANCES"] = ",".join(instances)
diff --git a/openpype/modules/default_modules/deadline/plugins/publish/submit_houdini_render_deadline.py b/openpype/modules/deadline/plugins/publish/submit_houdini_render_deadline.py
similarity index 89%
rename from openpype/modules/default_modules/deadline/plugins/publish/submit_houdini_render_deadline.py
rename to openpype/modules/deadline/plugins/publish/submit_houdini_render_deadline.py
index fa146c0d30b..beda7537235 100644
--- a/openpype/modules/default_modules/deadline/plugins/publish/submit_houdini_render_deadline.py
+++ b/openpype/modules/deadline/plugins/publish/submit_houdini_render_deadline.py
@@ -3,11 +3,11 @@
import getpass
import requests
-from avalon import api
-
import pyblish.api
-import hou
+# import hou ???
+
+from openpype.pipeline import legacy_io
class HoudiniSubmitRenderDeadline(pyblish.api.InstancePlugin):
@@ -50,8 +50,8 @@ def process(self, instance):
# StartFrame to EndFrame by byFrameStep
frames = "{start}-{end}x{step}".format(
- start=int(instance.data["startFrame"]),
- end=int(instance.data["endFrame"]),
+ start=int(instance.data["frameStart"]),
+ end=int(instance.data["frameEnd"]),
step=int(instance.data["byFrameStep"]),
)
@@ -71,7 +71,8 @@ def process(self, instance):
"UserName": deadline_user,
"Plugin": "Houdini",
- "Pool": "houdini_redshift", # todo: remove hardcoded pool
+ "Pool": instance.data.get("primaryPool"),
+ "secondaryPool": instance.data.get("secondaryPool"),
"Frames": frames,
"ChunkSize": instance.data.get("chunkSize", 10),
@@ -100,9 +101,14 @@ def process(self, instance):
# this application with so the Render Slave can build its own
# similar environment using it, e.g. "maya2018;vray4.x;yeti3.1.9"
"AVALON_TOOLS",
+ "OPENPYPE_VERSION"
]
+ # Add mongo url if it's enabled
+ if context.data.get("deadlinePassMongoUrl"):
+ keys.append("OPENPYPE_MONGO")
+
environment = dict({key: os.environ[key] for key in keys
- if key in os.environ}, **api.Session)
+ if key in os.environ}, **legacy_io.Session)
payload["JobInfo"].update({
"EnvironmentKeyValue%d" % index: "{key}={value}".format(
@@ -136,7 +142,7 @@ def process(self, instance):
def submit(self, instance, payload):
- AVALON_DEADLINE = api.Session.get("AVALON_DEADLINE",
+ AVALON_DEADLINE = legacy_io.Session.get("AVALON_DEADLINE",
"http://localhost:8082")
assert AVALON_DEADLINE, "Requires AVALON_DEADLINE"
diff --git a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py
new file mode 100644
index 00000000000..44f2b5b2b46
--- /dev/null
+++ b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py
@@ -0,0 +1,813 @@
+# -*- coding: utf-8 -*-
+"""Submitting render job to Deadline.
+
+This module is taking care of submitting job from Maya to Deadline. It
+creates job and set correct environments. Its behavior is controlled by
+``DEADLINE_REST_URL`` environment variable - pointing to Deadline Web Service
+and :data:`MayaSubmitDeadline.use_published` property telling Deadline to
+use published scene workfile or not.
+
+If ``vrscene`` or ``assscene`` are detected in families, it will first
+submit job to export these files and then dependent job to render them.
+
+Attributes:
+ payload_skeleton (dict): Skeleton payload data sent as job to Deadline.
+ Default values are for ``MayaBatch`` plugin.
+
+"""
+
+from __future__ import print_function
+import os
+import getpass
+import copy
+import re
+import hashlib
+from datetime import datetime
+import itertools
+from collections import OrderedDict
+
+import attr
+
+from maya import cmds
+
+from openpype.pipeline import legacy_io
+
+from openpype_modules.deadline import abstract_submit_deadline
+from openpype_modules.deadline.abstract_submit_deadline import DeadlineJobInfo
+
+
+@attr.s
+class MayaPluginInfo:
+ SceneFile = attr.ib(default=None) # Input
+ OutputFilePath = attr.ib(default=None) # Output directory and filename
+ OutputFilePrefix = attr.ib(default=None)
+ Version = attr.ib(default=None) # Mandatory for Deadline
+ UsingRenderLayers = attr.ib(default=True)
+ RenderLayer = attr.ib(default=None) # Render only this layer
+ Renderer = attr.ib(default=None)
+ ProjectPath = attr.ib(default=None) # Resolve relative references
+ RenderSetupIncludeLights = attr.ib(default=None) # Include all lights flag
+
+
+@attr.s
+class PythonPluginInfo:
+ ScriptFile = attr.ib()
+ Version = attr.ib(default="3.6")
+ Arguments = attr.ib(default=None)
+ SingleFrameOnly = attr.ib(default=None)
+
+
+@attr.s
+class VRayPluginInfo:
+ InputFilename = attr.ib(default=None) # Input
+ SeparateFilesPerFrame = attr.ib(default=None)
+ VRayEngine = attr.ib(default="V-Ray")
+ Width = attr.ib(default=None)
+ Height = attr.ib(default=None) # Mandatory for Deadline
+ OutputFilePath = attr.ib(default=True)
+ OutputFileName = attr.ib(default=None) # Render only this layer
+
+
+@attr.s
+class ArnoldPluginInfo:
+ ArnoldFile = attr.ib(default=None)
+
+
+class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline):
+
+ label = "Submit Render to Deadline"
+ hosts = ["maya"]
+ families = ["renderlayer"]
+ targets = ["local"]
+
+ tile_assembler_plugin = "OpenPypeTileAssembler"
+ priority = 50
+ tile_priority = 50
+ limit = [] # limit groups
+ jobInfo = {}
+ pluginInfo = {}
+ group = "none"
+
+ def get_job_info(self):
+ job_info = DeadlineJobInfo(Plugin="MayaBatch")
+
+ # todo: test whether this works for existing production cases
+ # where custom jobInfo was stored in the project settings
+ job_info.update(self.jobInfo)
+
+ instance = self._instance
+ context = instance.context
+
+ # Always use the original work file name for the Job name even when
+ # rendering is done from the published Work File. The original work
+ # file name is clearer because it can also have subversion strings,
+ # etc. which are stripped for the published file.
+ src_filepath = context.data["currentFile"]
+ src_filename = os.path.basename(src_filepath)
+
+ job_info.Name = "%s - %s" % (src_filename, instance.name)
+ job_info.BatchName = src_filename
+ job_info.Plugin = instance.data.get("mayaRenderPlugin", "MayaBatch")
+ job_info.UserName = context.data.get("deadlineUser", getpass.getuser())
+
+ # Deadline requires integers in frame range
+ frames = "{start}-{end}x{step}".format(
+ start=int(instance.data["frameStartHandle"]),
+ end=int(instance.data["frameEndHandle"]),
+ step=int(instance.data["byFrameStep"]),
+ )
+ job_info.Frames = frames
+
+ job_info.Pool = instance.data.get("primaryPool")
+ job_info.SecondaryPool = instance.data.get("secondaryPool")
+ job_info.ChunkSize = instance.data.get("chunkSize", 10)
+ job_info.Comment = context.data.get("comment")
+ job_info.Priority = instance.data.get("priority", self.priority)
+ job_info.FramesPerTask = instance.data.get("framesPerTask", 1)
+
+ if self.group != "none" and self.group:
+ job_info.Group = self.group
+
+ if self.limit:
+ job_info.LimitGroups = ",".join(self.limit)
+
+ # Add options from RenderGlobals
+ render_globals = instance.data.get("renderGlobals", {})
+ job_info.update(render_globals)
+
+ keys = [
+ "FTRACK_API_KEY",
+ "FTRACK_API_USER",
+ "FTRACK_SERVER",
+ "OPENPYPE_SG_USER",
+ "AVALON_PROJECT",
+ "AVALON_ASSET",
+ "AVALON_TASK",
+ "AVALON_APP_NAME",
+ "OPENPYPE_DEV",
+ "OPENPYPE_VERSION"
+ ]
+ # Add mongo url if it's enabled
+ if self._instance.context.data.get("deadlinePassMongoUrl"):
+ keys.append("OPENPYPE_MONGO")
+
+ environment = dict({key: os.environ[key] for key in keys
+ if key in os.environ}, **legacy_io.Session)
+
+ for key in keys:
+ value = environment.get(key)
+ if not value:
+ continue
+ job_info.EnvironmentKeyValue[key] = value
+
+ # to recognize job from PYPE for turning Event On/Off
+ job_info.EnvironmentKeyValue["OPENPYPE_RENDER_JOB"] = "1"
+ job_info.EnvironmentKeyValue["OPENPYPE_LOG_NO_COLORS"] = "1"
+
+ # Adding file dependencies.
+ if self.asset_dependencies:
+ dependencies = instance.context.data["fileDependencies"]
+ dependencies.append(context.data["currentFile"])
+ for dependency in dependencies:
+ job_info.AssetDependency += dependency
+
+ # Add list of expected files to job
+ # ---------------------------------
+ exp = instance.data.get("expectedFiles")
+ for filepath in self._iter_expected_files(exp):
+ job_info.OutputDirectory += os.path.dirname(filepath)
+ job_info.OutputFilename += os.path.basename(filepath)
+
+ return job_info
+
+ def get_plugin_info(self):
+
+ instance = self._instance
+ context = instance.context
+
+ plugin_info = MayaPluginInfo(
+ SceneFile=self.scene_path,
+ Version=cmds.about(version=True),
+ RenderLayer=instance.data['setMembers'],
+ Renderer=instance.data["renderer"],
+ RenderSetupIncludeLights=instance.data.get("renderSetupIncludeLights"), # noqa
+ ProjectPath=context.data["workspaceDir"],
+ UsingRenderLayers=True,
+ )
+
+ plugin_payload = attr.asdict(plugin_info)
+
+ # Patching with pluginInfo from settings
+ for key, value in self.pluginInfo.items():
+ plugin_payload[key] = value
+
+ return plugin_payload
+
+ def process_submission(self):
+
+ instance = self._instance
+ context = instance.context
+
+ filepath = self.scene_path # publish if `use_publish` else workfile
+
+ # TODO: Avoid the need for this logic here, needed for submit publish
+ # Store output dir for unified publisher (filesequence)
+ expected_files = instance.data["expectedFiles"]
+ first_file = next(self._iter_expected_files(expected_files))
+ output_dir = os.path.dirname(first_file)
+ instance.data["outputDir"] = output_dir
+ instance.data["toBeRenderedOn"] = "deadline"
+
+ # Patch workfile (only when use_published is enabled)
+ if self.use_published:
+ self._patch_workfile()
+
+ # Gather needed data ------------------------------------------------
+ workspace = context.data["workspaceDir"]
+ default_render_file = instance.context.data.get('project_settings')\
+ .get('maya')\
+ .get('RenderSettings')\
+ .get('default_render_image_folder')
+ filename = os.path.basename(filepath)
+ dirname = os.path.join(workspace, default_render_file)
+
+ # Fill in common data to payload ------------------------------------
+ # TODO: Replace these with collected data from CollectRender
+ payload_data = {
+ "filename": filename,
+ "dirname": dirname,
+ }
+
+ # Submit preceding export jobs -------------------------------------
+ export_job = None
+ assert not all(x in instance.data["families"]
+ for x in ['vrayscene', 'assscene']), (
+ "Vray Scene and Ass Scene options are mutually exclusive")
+
+ if "vrayscene" in instance.data["families"]:
+ self.log.debug("Submitting V-Ray scene render..")
+ vray_export_payload = self._get_vray_export_payload(payload_data)
+ export_job = self.submit(vray_export_payload)
+
+ payload = self._get_vray_render_payload(payload_data)
+
+ elif "assscene" in instance.data["families"]:
+ self.log.debug("Submitting Arnold .ass standalone render..")
+ ass_export_payload = self._get_arnold_export_payload(payload_data)
+ export_job = self.submit(ass_export_payload)
+
+ payload = self._get_arnold_render_payload(payload_data)
+ else:
+ self.log.debug("Submitting MayaBatch render..")
+ payload = self._get_maya_payload(payload_data)
+
+ # Add export job as dependency --------------------------------------
+ if export_job:
+ job_info, _ = payload
+ job_info.JobDependency = export_job
+
+ if instance.data.get("tileRendering"):
+ # Prepare tiles data
+ self._tile_render(payload)
+ else:
+ # Submit main render job
+ job_info, plugin_info = payload
+ self.submit(self.assemble_payload(job_info, plugin_info))
+
+ def _tile_render(self, payload):
+ """Submit as tile render per frame with dependent assembly jobs."""
+
+ # As collected by super process()
+ instance = self._instance
+
+ payload_job_info, payload_plugin_info = payload
+ job_info = copy.deepcopy(payload_job_info)
+ plugin_info = copy.deepcopy(payload_plugin_info)
+
+ # if we have sequence of files, we need to create tile job for
+ # every frame
+ job_info.TileJob = True
+ job_info.TileJobTilesInX = instance.data.get("tilesX")
+ job_info.TileJobTilesInY = instance.data.get("tilesY")
+
+ tiles_count = job_info.TileJobTilesInX * job_info.TileJobTilesInY
+
+ plugin_info["ImageHeight"] = instance.data.get("resolutionHeight")
+ plugin_info["ImageWidth"] = instance.data.get("resolutionWidth")
+ plugin_info["RegionRendering"] = True
+
+ R_FRAME_NUMBER = re.compile(
+ r".+\.(?P/_/
+ "shot010_v006/shot010_v006_CHARS/CHARS_0001.vrscene"
+ Returns:
+ str
+
+ """
+
+ # "vrayscene//_/"
+ vray_settings = cmds.ls(type="VRaySettingsNode")
+ node = vray_settings[0]
+ template = cmds.getAttr("{}.vrscene_filename".format(node))
+ scene, _ = os.path.splitext(self.scene_path)
+
+ def smart_replace(string, key_values):
+ new_string = string
+ for key, value in key_values.items():
+ new_string = new_string.replace(key, value)
+ return new_string
+
+ # Get workfile scene path without extension to format vrscene_filename
+ scene_filename = os.path.basename(self.scene_path)
+ scene_filename_no_ext, _ = os.path.splitext(scene_filename)
+
+ layer = self._instance.data['setMembers']
+
+ # Reformat without tokens
+ output_path = smart_replace(
+ template,
+ {"": scene_filename_no_ext,
+ "": layer})
+
+ start_frame = int(self._instance.data["frameStartHandle"])
+ workspace = self._instance.context.data["workspace"]
+ filename_zero = "{}_{:04d}.vrscene".format(output_path, start_frame)
+ filepath_zero = os.path.join(workspace, filename_zero)
+
+ return filepath_zero.replace("\\", "/")
+
+ def _patch_workfile(self):
+ """Patch Maya scene.
+
+ This will take list of patches (lines to add) and apply them to
+ *published* Maya scene file (that is used later for rendering).
+
+ Patches are dict with following structure::
+ {
+ "name": "Name of patch",
+ "regex": "regex of line before patch",
+ "line": "line to insert"
+ }
+
+ """
+ project_settings = self._instance.context.data["project_settings"]
+ patches = (
+ project_settings.get(
+ "deadline", {}).get(
+ "publish", {}).get(
+ "MayaSubmitDeadline", {}).get(
+ "scene_patches", {})
+ )
+ if not patches:
+ return
+
+ if not os.path.splitext(self.scene_path)[1].lower() != ".ma":
+ self.log.debug("Skipping workfile patch since workfile is not "
+ ".ma file")
+ return
+
+ compiled_regex = [re.compile(p["regex"]) for p in patches]
+ with open(self.scene_path, "r+") as pf:
+ scene_data = pf.readlines()
+ for ln, line in enumerate(scene_data):
+ for i, r in enumerate(compiled_regex):
+ if re.match(r, line):
+ scene_data.insert(ln + 1, patches[i]["line"])
+ pf.seek(0)
+ pf.writelines(scene_data)
+ pf.truncate()
+ self.log.info("Applied {} patch to scene.".format(
+ patches[i]["name"]
+ ))
+
+ def _job_info_label(self, label):
+ return "{label} {job.Name} [{start}-{end}]".format(
+ label=label,
+ job=self.job_info,
+ start=int(self._instance.data["frameStartHandle"]),
+ end=int(self._instance.data["frameEndHandle"]),
+ )
+
+ @staticmethod
+ def _iter_expected_files(exp):
+ if isinstance(exp[0], dict):
+ for _aov, files in exp[0].items():
+ for file in files:
+ yield file
+ else:
+ for file in exp:
+ yield file
+
+
+def _format_tiles(
+ filename, index, tiles_x, tiles_y,
+ width, height, prefix):
+ """Generate tile entries for Deadline tile job.
+
+ Returns two dictionaries - one that can be directly used in Deadline
+ job, second that can be used for Deadline Assembly job configuration
+ file.
+
+ This will format tile names:
+
+ Example::
+ {
+ "OutputFilename0Tile0": "_tile_1x1_4x4_Main_beauty.1001.exr",
+ "OutputFilename0Tile1": "_tile_2x1_4x4_Main_beauty.1001.exr"
+ }
+
+ And add tile prefixes like:
+
+ Example::
+ Image prefix is:
+ `maya///_`
+
+ Result for tile 0 for 4x4 will be:
+ `maya///_tile_1x1_4x4__`
+
+ Calculating coordinates is tricky as in Job they are defined as top,
+ left, bottom, right with zero being in top-left corner. But Assembler
+ configuration file takes tile coordinates as X, Y, Width and Height and
+ zero is bottom left corner.
+
+ Args:
+ filename (str): Filename to process as tiles.
+ index (int): Index of that file if it is sequence.
+ tiles_x (int): Number of tiles in X.
+ tiles_y (int): Number of tiles in Y.
+ width (int): Width resolution of final image.
+ height (int): Height resolution of final image.
+ prefix (str): Image prefix.
+
+ Returns:
+ (dict, dict): Tuple of two dictionaries - first can be used to
+ extend JobInfo, second has tiles x, y, width and height
+ used for assembler configuration.
+
+ """
+ # Math used requires integers for correct output - as such
+ # we ensure our inputs are correct.
+ assert type(tiles_x) is int, "tiles_x must be an integer"
+ assert type(tiles_y) is int, "tiles_y must be an integer"
+ assert type(width) is int, "width must be an integer"
+ assert type(height) is int, "height must be an integer"
+
+ out = {"JobInfo": {}, "PluginInfo": {}}
+ cfg = OrderedDict()
+ w_space = width // tiles_x
+ h_space = height // tiles_y
+
+ cfg["TilesCropped"] = "False"
+
+ tile = 0
+ for tile_x in range(1, tiles_x + 1):
+ for tile_y in reversed(range(1, tiles_y + 1)):
+ tile_prefix = "_tile_{}x{}_{}x{}_".format(
+ tile_x, tile_y,
+ tiles_x,
+ tiles_y
+ )
+
+ new_filename = "{}/{}{}".format(
+ os.path.dirname(filename),
+ tile_prefix,
+ os.path.basename(filename)
+ )
+
+ top = height - (tile_y * h_space)
+ bottom = height - ((tile_y - 1) * h_space) - 1
+ left = (tile_x - 1) * w_space
+ right = (tile_x * w_space) - 1
+
+ # Job info
+ out["JobInfo"]["OutputFilename{}Tile{}".format(index, tile)] = new_filename # noqa: E501
+
+ # Plugin Info
+ out["PluginInfo"]["RegionPrefix{}".format(str(tile))] = \
+ "/{}".format(tile_prefix).join(prefix.rsplit("/", 1))
+ out["PluginInfo"]["RegionTop{}".format(tile)] = top
+ out["PluginInfo"]["RegionBottom{}".format(tile)] = bottom
+ out["PluginInfo"]["RegionLeft{}".format(tile)] = left
+ out["PluginInfo"]["RegionRight{}".format(tile)] = right
+
+ # Tile config
+ cfg["Tile{}".format(tile)] = new_filename
+ cfg["Tile{}Tile".format(tile)] = new_filename
+ cfg["Tile{}FileName".format(tile)] = new_filename
+ cfg["Tile{}X".format(tile)] = left
+ cfg["Tile{}Y".format(tile)] = top
+ cfg["Tile{}Width".format(tile)] = w_space
+ cfg["Tile{}Height".format(tile)] = h_space
+
+ tile += 1
+
+ return out, cfg
diff --git a/openpype/modules/deadline/plugins/publish/submit_maya_remote_publish_deadline.py b/openpype/modules/deadline/plugins/publish/submit_maya_remote_publish_deadline.py
new file mode 100644
index 00000000000..38ae5d2f7f9
--- /dev/null
+++ b/openpype/modules/deadline/plugins/publish/submit_maya_remote_publish_deadline.py
@@ -0,0 +1,136 @@
+import os
+import requests
+
+from maya import cmds
+
+from openpype.pipeline import legacy_io, PublishXmlValidationError
+from openpype.settings import get_project_settings
+
+import pyblish.api
+
+
+class MayaSubmitRemotePublishDeadline(pyblish.api.InstancePlugin):
+ """Submit Maya scene to perform a local publish in Deadline.
+
+ Publishing in Deadline can be helpful for scenes that publish very slow.
+ This way it can process in the background on another machine without the
+ Artist having to wait for the publish to finish on their local machine.
+
+ Submission is done through the Deadline Web Service. DL then triggers
+ `openpype/scripts/remote_publish.py`.
+
+ Each publishable instance creates its own full publish job.
+
+ Different from `ProcessSubmittedJobOnFarm` which creates publish job
+ depending on metadata json containing context and instance data of
+ rendered files.
+ """
+
+ label = "Submit Scene to Deadline"
+ order = pyblish.api.IntegratorOrder
+ hosts = ["maya"]
+ families = ["publish.farm"]
+ targets = ["local"]
+
+ def process(self, instance):
+ project_name = instance.context.data["projectName"]
+ # TODO settings can be received from 'context.data["project_settings"]'
+ settings = get_project_settings(project_name)
+ # use setting for publish job on farm, no reason to have it separately
+ deadline_publish_job_sett = (settings["deadline"]
+ ["publish"]
+ ["ProcessSubmittedJobOnFarm"])
+
+ # Ensure no errors so far
+ if not (all(result["success"]
+ for result in instance.context.data["results"])):
+ raise PublishXmlValidationError("Publish process has errors")
+
+ if not instance.data["publish"]:
+ self.log.warning("No active instances found. "
+ "Skipping submission..")
+ return
+
+ scene = instance.context.data["currentFile"]
+ scenename = os.path.basename(scene)
+
+ job_name = "{scene} [PUBLISH]".format(scene=scenename)
+ batch_name = "{code} - {scene}".format(code=project_name,
+ scene=scenename)
+
+ # Generate the payload for Deadline submission
+ payload = {
+ "JobInfo": {
+ "Plugin": "MayaBatch",
+ "BatchName": batch_name,
+ "Name": job_name,
+ "UserName": instance.context.data["user"],
+ "Comment": instance.context.data.get("comment", ""),
+ # "InitialStatus": state
+ "Department": deadline_publish_job_sett["deadline_department"],
+ "ChunkSize": deadline_publish_job_sett["deadline_chunk_size"],
+ "Priority": deadline_publish_job_sett["deadline_priority"],
+ "Group": deadline_publish_job_sett["deadline_group"],
+ "Pool": deadline_publish_job_sett["deadline_pool"],
+ },
+ "PluginInfo": {
+
+ "Build": None, # Don't force build
+ "StrictErrorChecking": True,
+ "ScriptJob": True,
+
+ # Inputs
+ "SceneFile": scene,
+ "ScriptFilename": "{OPENPYPE_REPOS_ROOT}/openpype/scripts/remote_publish.py", # noqa
+
+ # Mandatory for Deadline
+ "Version": cmds.about(version=True),
+
+ # Resolve relative references
+ "ProjectPath": cmds.workspace(query=True,
+ rootDirectory=True),
+
+ },
+
+ # Mandatory for Deadline, may be empty
+ "AuxFiles": []
+ }
+
+ # Include critical environment variables with submission + api.Session
+ keys = [
+ "FTRACK_API_USER",
+ "FTRACK_API_KEY",
+ "FTRACK_SERVER",
+ "OPENPYPE_VERSION"
+ ]
+ environment = dict({key: os.environ[key] for key in keys
+ if key in os.environ}, **legacy_io.Session)
+
+ # TODO replace legacy_io with context.data
+ environment["AVALON_PROJECT"] = project_name
+ environment["AVALON_ASSET"] = legacy_io.Session["AVALON_ASSET"]
+ environment["AVALON_TASK"] = legacy_io.Session["AVALON_TASK"]
+ environment["AVALON_APP_NAME"] = os.environ.get("AVALON_APP_NAME")
+ environment["OPENPYPE_LOG_NO_COLORS"] = "1"
+ environment["OPENPYPE_REMOTE_JOB"] = "1"
+ environment["OPENPYPE_USERNAME"] = instance.context.data["user"]
+ environment["OPENPYPE_PUBLISH_SUBSET"] = instance.data["subset"]
+ environment["OPENPYPE_REMOTE_PUBLISH"] = "1"
+
+ payload["JobInfo"].update({
+ "EnvironmentKeyValue%d" % index: "{key}={value}".format(
+ key=key,
+ value=environment[key]
+ ) for index, key in enumerate(environment)
+ })
+
+ self.log.info("Submitting Deadline job ...")
+ deadline_url = instance.context.data["defaultDeadline"]
+ # if custom one is set in instance, use that
+ if instance.data.get("deadlineUrl"):
+ deadline_url = instance.data.get("deadlineUrl")
+ assert deadline_url, "Requires Deadline Webservice URL"
+ url = "{}/api/jobs".format(deadline_url)
+ response = requests.post(url, json=payload, timeout=10)
+ if not response.ok:
+ raise Exception(response.text)
diff --git a/openpype/modules/default_modules/deadline/plugins/publish/submit_nuke_deadline.py b/openpype/modules/deadline/plugins/publish/submit_nuke_deadline.py
similarity index 83%
rename from openpype/modules/default_modules/deadline/plugins/publish/submit_nuke_deadline.py
rename to openpype/modules/deadline/plugins/publish/submit_nuke_deadline.py
index ae9cd985ebe..b09d2935abf 100644
--- a/openpype/modules/default_modules/deadline/plugins/publish/submit_nuke_deadline.py
+++ b/openpype/modules/deadline/plugins/publish/submit_nuke_deadline.py
@@ -4,10 +4,10 @@
import getpass
import requests
-
-from avalon import api
import pyblish.api
+
import nuke
+from openpype.pipeline import legacy_io
class NukeSubmitDeadline(pyblish.api.InstancePlugin):
@@ -23,12 +23,12 @@ class NukeSubmitDeadline(pyblish.api.InstancePlugin):
hosts = ["nuke", "nukestudio"]
families = ["render.farm", "prerender.farm"]
optional = True
+ targets = ["local"]
# presets
priority = 50
chunk_size = 1
- primary_pool = ""
- secondary_pool = ""
+ concurrent_tasks = 1
group = ""
department = ""
limit_groups = {}
@@ -55,8 +55,8 @@ def process(self, instance):
self._ver = re.search(r"\d+\.\d+", context.data.get("hostVersion"))
self._deadline_user = context.data.get(
"deadlineUser", getpass.getuser())
- self._frame_start = int(instance.data["frameStartHandle"])
- self._frame_end = int(instance.data["frameEndHandle"])
+ submit_frame_start = int(instance.data["frameStartHandle"])
+ submit_frame_end = int(instance.data["frameEndHandle"])
# get output path
render_path = instance.data['path']
@@ -80,15 +80,14 @@ def process(self, instance):
"Using published scene for render {}".format(script_path)
)
- # exception for slate workflow
- if "slate" in instance.data["families"]:
- self._frame_start -= 1
-
- response = self.payload_submit(instance,
- script_path,
- render_path,
- node.name()
- )
+ response = self.payload_submit(
+ instance,
+ script_path,
+ render_path,
+ node.name(),
+ submit_frame_start,
+ submit_frame_end
+ )
# Store output dir for unified publisher (filesequence)
instance.data["deadlineSubmissionJob"] = response.json()
instance.data["outputDir"] = os.path.dirname(
@@ -101,15 +100,13 @@ def process(self, instance):
script_path = baking_script["bakeScriptPath"]
exe_node_name = baking_script["bakeWriteNodeName"]
- # exception for slate workflow
- if "slate" in instance.data["families"]:
- self._frame_start += 1
-
resp = self.payload_submit(
instance,
script_path,
render_path,
exe_node_name,
+ submit_frame_start,
+ submit_frame_end,
response.json()
)
@@ -117,6 +114,13 @@ def process(self, instance):
instance.data["deadlineSubmissionJob"] = resp.json()
instance.data["publishJobState"] = "Suspended"
+ # add to list of job Id
+ if not instance.data.get("bakingSubmissionJobs"):
+ instance.data["bakingSubmissionJobs"] = []
+
+ instance.data["bakingSubmissionJobs"].append(
+ resp.json()["_id"])
+
# redefinition of families
if "render.farm" in families:
instance.data['family'] = 'write'
@@ -126,13 +130,16 @@ def process(self, instance):
families.insert(0, "prerender")
instance.data["families"] = families
- def payload_submit(self,
- instance,
- script_path,
- render_path,
- exe_node_name,
- responce_data=None
- ):
+ def payload_submit(
+ self,
+ instance,
+ script_path,
+ render_path,
+ exe_node_name,
+ start_frame,
+ end_frame,
+ responce_data=None
+ ):
render_dir = os.path.normpath(os.path.dirname(render_path))
script_name = os.path.basename(script_path)
jobname = "%s - %s" % (script_name, instance.name)
@@ -149,11 +156,16 @@ def payload_submit(self,
pass
# define chunk and priority
- chunk_size = instance.data.get("deadlineChunkSize")
+ chunk_size = instance.data["deadlineChunkSize"]
if chunk_size == 0 and self.chunk_size:
chunk_size = self.chunk_size
- priority = instance.data.get("deadlinePriority")
+ # define chunk and priority
+ concurrent_tasks = instance.data["deadlineConcurrentTasks"]
+ if concurrent_tasks == 0 and self.concurrent_tasks:
+ concurrent_tasks = self.concurrent_tasks
+
+ priority = instance.data["deadlinePriority"]
if not priority:
priority = self.priority
@@ -177,16 +189,18 @@ def payload_submit(self,
"Priority": priority,
"ChunkSize": chunk_size,
+ "ConcurrentTasks": concurrent_tasks,
+
"Department": self.department,
- "Pool": self.primary_pool,
- "SecondaryPool": self.secondary_pool,
+ "Pool": instance.data.get("primaryPool"),
+ "SecondaryPool": instance.data.get("secondaryPool"),
"Group": self.group,
"Plugin": "Nuke",
"Frames": "{start}-{end}".format(
- start=self._frame_start,
- end=self._frame_end
+ start=start_frame,
+ end=end_frame
),
"Comment": self._comment,
@@ -236,7 +250,6 @@ def payload_submit(self,
keys = [
"PYTHONPATH",
"PATH",
- "AVALON_SCHEMA",
"AVALON_PROJECT",
"AVALON_ASSET",
"AVALON_TASK",
@@ -247,14 +260,19 @@ def payload_submit(self,
"PYBLISHPLUGINPATH",
"NUKE_PATH",
"TOOL_ENV",
- "FOUNDRY_LICENSE"
+ "FOUNDRY_LICENSE",
+ "OPENPYPE_VERSION"
]
+ # Add mongo url if it's enabled
+ if instance.context.data.get("deadlinePassMongoUrl"):
+ keys.append("OPENPYPE_MONGO")
+
# add allowed keys from preset if any
if self.env_allowed_keys:
keys += self.env_allowed_keys
environment = dict({key: os.environ[key] for key in keys
- if key in os.environ}, **api.Session)
+ if key in os.environ}, **legacy_io.Session)
for _path in os.environ:
if _path.lower().startswith('openpype_'):
@@ -283,7 +301,13 @@ def payload_submit(self,
self.log.info(json.dumps(payload, indent=4, sort_keys=True))
# adding expectied files to instance.data
- self.expected_files(instance, render_path)
+ self.expected_files(
+ instance,
+ render_path,
+ start_frame,
+ end_frame
+ )
+
self.log.debug("__ expectedFiles: `{}`".format(
instance.data["expectedFiles"]))
response = requests.post(self.deadline_url, json=payload, timeout=10)
@@ -329,15 +353,19 @@ def preview_fname(self, path):
self.log.debug("_ path: `{}`".format(path))
return path
- def expected_files(self,
- instance,
- path):
+ def expected_files(
+ self,
+ instance,
+ path,
+ start_frame,
+ end_frame
+ ):
""" Create expected files in instance data
"""
if not instance.data.get("expectedFiles"):
instance.data["expectedFiles"] = []
- dir = os.path.dirname(path)
+ dirname = os.path.dirname(path)
file = os.path.basename(path)
if "#" in file:
@@ -349,9 +377,12 @@ def expected_files(self,
instance.data["expectedFiles"].append(path)
return
- for i in range(self._frame_start, (self._frame_end + 1)):
+ if instance.data.get("slate"):
+ start_frame -= 1
+
+ for i in range(start_frame, (end_frame + 1)):
instance.data["expectedFiles"].append(
- os.path.join(dir, (file % i)).replace("\\", "/"))
+ os.path.join(dirname, (file % i)).replace("\\", "/"))
def get_limit_groups(self):
"""Search for limit group nodes and return group name.
diff --git a/openpype/modules/default_modules/deadline/plugins/publish/submit_publish_job.py b/openpype/modules/deadline/plugins/publish/submit_publish_job.py
similarity index 84%
rename from openpype/modules/default_modules/deadline/plugins/publish/submit_publish_job.py
rename to openpype/modules/deadline/plugins/publish/submit_publish_job.py
index 516bd755d03..aba505b3c64 100644
--- a/openpype/modules/default_modules/deadline/plugins/publish/submit_publish_job.py
+++ b/openpype/modules/deadline/plugins/publish/submit_publish_job.py
@@ -7,23 +7,38 @@
from copy import copy, deepcopy
import requests
import clique
-import openpype.api
-
-from avalon import api, io
import pyblish.api
+from openpype.client import (
+ get_last_version_by_subset_name,
+ get_representations,
+)
+from openpype.pipeline import (
+ get_representation_path,
+ legacy_io,
+)
+from openpype.pipeline.farm.patterning import match_aov_pattern
-def get_resources(version, extension=None):
- """Get the files from the specific version."""
- query = {"type": "representation", "parent": version["_id"]}
- if extension:
- query["name"] = extension
- representation = io.find_one(query)
- assert representation, "This is a bug"
+def get_resources(project_name, version, extension=None):
+ """Get the files from the specific version."""
- directory = api.get_representation_path(representation)
+ # TODO this functions seems to be weird
+ # - it's looking for representation with one extension or first (any)
+ # representation from a version?
+ # - not sure how this should work, maybe it does for specific use cases
+ # but probably can't be used for all resources from 2D workflows
+ extensions = None
+ if extension:
+ extensions = [extension]
+ repre_docs = list(get_representations(
+ project_name, version_ids=[version["_id"]], extensions=extensions
+ ))
+ assert repre_docs, "This is a bug"
+
+ representation = repre_docs[0]
+ directory = get_representation_path(representation)
print("Source: ", directory)
resources = sorted(
[
@@ -99,13 +114,14 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin):
order = pyblish.api.IntegratorOrder + 0.2
icon = "tractor"
deadline_plugin = "OpenPype"
+ targets = ["local"]
hosts = ["fusion", "maya", "nuke", "celaction", "aftereffects", "harmony"]
families = ["render.farm", "prerender.farm",
"renderlayer", "imagesequence", "vrayscene"]
- aov_filter = {"maya": [r".*(?:[\._-])*([Bb]eauty)(?:[\.|_])*.*"],
+ aov_filter = {"maya": [r".*([Bb]eauty).*"],
"aftereffects": [r".*"], # for everything from AE
"harmony": [r".*"], # for everything from AE
"celaction": [r".*"]}
@@ -124,10 +140,12 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin):
"OPENPYPE_LOG_NO_COLORS",
"OPENPYPE_USERNAME",
"OPENPYPE_RENDER_JOB",
- "OPENPYPE_PUBLISH_JOB"
+ "OPENPYPE_PUBLISH_JOB",
+ "OPENPYPE_MONGO",
+ "OPENPYPE_VERSION"
]
- # custom deadline atributes
+ # custom deadline attributes
deadline_department = ""
deadline_pool = ""
deadline_pool_secondary = ""
@@ -141,7 +159,7 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin):
# mapping of instance properties to be transfered to new instance for every
# specified family
instance_transfer = {
- "slate": ["slateFrame"],
+ "slate": ["slateFrames", "slate"],
"review": ["lutPath"],
"render2d": ["bakingNukeScripts", "version"],
"renderlayer": ["convertToScanline"]
@@ -219,20 +237,28 @@ def _submit_deadline_post_job(self, instance, job, instances):
self._create_metadata_path(instance)
environment = job["Props"].get("Env", {})
- environment["AVALON_PROJECT"] = io.Session["AVALON_PROJECT"]
- environment["AVALON_ASSET"] = io.Session["AVALON_ASSET"]
- environment["AVALON_TASK"] = io.Session["AVALON_TASK"]
+ environment["AVALON_PROJECT"] = legacy_io.Session["AVALON_PROJECT"]
+ environment["AVALON_ASSET"] = legacy_io.Session["AVALON_ASSET"]
+ environment["AVALON_TASK"] = legacy_io.Session["AVALON_TASK"]
environment["AVALON_APP_NAME"] = os.environ.get("AVALON_APP_NAME")
environment["OPENPYPE_LOG_NO_COLORS"] = "1"
environment["OPENPYPE_USERNAME"] = instance.context.data["user"]
environment["OPENPYPE_PUBLISH_JOB"] = "1"
environment["OPENPYPE_RENDER_JOB"] = "0"
+ # Add mongo url if it's enabled
+ if instance.context.data.get("deadlinePassMongoUrl"):
+ mongo_url = os.environ.get("OPENPYPE_MONGO")
+ if mongo_url:
+ environment["OPENPYPE_MONGO"] = mongo_url
+
+ priority = self.deadline_priority or instance.data.get("priority", 50)
args = [
+ "--headless",
'publish',
roothless_metadata_path,
"--targets", "deadline",
- "--targets", "filesequence"
+ "--targets", "farm"
]
# Generate the payload for Deadline submission
@@ -246,11 +272,11 @@ def _submit_deadline_post_job(self, instance, job, instances):
"Department": self.deadline_department,
"ChunkSize": self.deadline_chunk_size,
- "Priority": job["Props"]["Pri"],
+ "Priority": priority,
"Group": self.deadline_group,
- "Pool": self.deadline_pool,
- "SecondaryPool": self.deadline_pool_secondary,
+ "Pool": instance.data.get("primaryPool"),
+ "SecondaryPool": instance.data.get("secondaryPool"),
"OutputDirectory0": output_dir
},
@@ -270,21 +296,30 @@ def _submit_deadline_post_job(self, instance, job, instances):
for assembly_id in instance.data.get("assemblySubmissionJobs"):
payload["JobInfo"]["JobDependency{}".format(job_index)] = assembly_id # noqa: E501
job_index += 1
+ elif instance.data.get("bakingSubmissionJobs"):
+ self.log.info("Adding baking submission jobs as dependencies...")
+ job_index = 0
+ for assembly_id in instance.data["bakingSubmissionJobs"]:
+ payload["JobInfo"]["JobDependency{}".format(job_index)] = assembly_id # noqa: E501
+ job_index += 1
else:
payload["JobInfo"]["JobDependency0"] = job["_id"]
- i = 0
- for index, key in enumerate(environment):
+ if instance.data.get("suspend_publish"):
+ payload["JobInfo"]["InitialStatus"] = "Suspended"
+
+ index = 0
+ for key in environment:
if key.upper() in self.enviro_filter:
payload["JobInfo"].update(
{
"EnvironmentKeyValue%d"
- % i: "{key}={value}".format(
+ % index: "{key}={value}".format(
key=key, value=environment[key]
)
}
)
- i += 1
+ index += 1
# remove secondary pool
payload["JobInfo"].pop("SecondaryPool", None)
@@ -311,15 +346,23 @@ def _copy_extend_frames(self, instance, representation):
import speedcopy
self.log.info("Preparing to copy ...")
- start = instance.data.get("startFrame")
- end = instance.data.get("endFrame")
+ start = instance.data.get("frameStart")
+ end = instance.data.get("frameEnd")
+ project_name = legacy_io.active_project()
# get latest version of subset
# this will stop if subset wasn't published yet
- version = openpype.api.get_latest_version(instance.data.get("asset"),
- instance.data.get("subset"))
+ project_name = legacy_io.active_project()
+ version = get_last_version_by_subset_name(
+ project_name,
+ instance.data.get("subset"),
+ asset_name=instance.data.get("asset")
+ )
+
# get its files based on extension
- subset_resources = get_resources(version, representation.get("ext"))
+ subset_resources = get_resources(
+ project_name, version, representation.get("ext")
+ )
r_col, _ = clique.assemble(subset_resources)
# if override remove all frames we are expecting to be rendered
@@ -439,16 +482,19 @@ def _create_instances_for_aov(self, instance_data, exp_files):
app = os.environ.get("AVALON_APP", "")
preview = False
- if app in self.aov_filter.keys():
- for aov_pattern in self.aov_filter[app]:
- if re.match(aov_pattern, aov):
- preview = True
- break
+ if isinstance(col, list):
+ render_file_name = os.path.basename(col[0])
+ else:
+ render_file_name = os.path.basename(col)
+ aov_patterns = self.aov_filter
+ preview = match_aov_pattern(app, aov_patterns, render_file_name)
+
+ # toggle preview on if multipart is on
if instance_data.get("multipartExr"):
preview = True
- new_instance = copy(instance_data)
+ new_instance = deepcopy(instance_data)
new_instance["subset"] = subset_name
new_instance["subsetGroup"] = group_name
if preview:
@@ -501,8 +547,8 @@ def _get_representations(self, instance, exp_files):
most cases, but if not - we create representation from each of them.
Arguments:
- instance (pyblish.plugin.Instance): instance for which we are
- setting representations
+ instance (dict): instance data for which we are
+ setting representations
exp_files (list): list of expected files
Returns:
@@ -510,31 +556,29 @@ def _get_representations(self, instance, exp_files):
"""
representations = []
+ host_name = os.environ.get("AVALON_APP", "")
collections, remainders = clique.assemble(exp_files)
- bake_renders = instance.get("bakingNukeScripts", [])
# create representation for every collected sequento ce
for collection in collections:
ext = collection.tail.lstrip(".")
preview = False
- # if filtered aov name is found in filename, toggle it for
- # preview video rendering
- for app in self.aov_filter.keys():
- if os.environ.get("AVALON_APP", "") == app:
- for aov in self.aov_filter[app]:
- if re.match(
- aov,
- list(collection)[0]
- ):
- preview = True
- break
-
- if bake_renders:
- preview = False
-
- # toggle preview on if multipart is on
- if instance.get("multipartExr", False):
- preview = True
+ # TODO 'useSequenceForReview' is temporary solution which does
+ # not work for 100% of cases. We must be able to tell what
+ # expected files contains more explicitly and from what
+ # should be review made.
+ # - "review" tag is never added when is set to 'False'
+ if instance["useSequenceForReview"]:
+ # toggle preview on if multipart is on
+ if instance.get("multipartExr", False):
+ preview = True
+ else:
+ render_file_name = list(collection)[0]
+ # if filtered aov name is found in filename, toggle it for
+ # preview video rendering
+ preview = match_aov_pattern(
+ host_name, self.aov_filter, render_file_name
+ )
staging = os.path.dirname(list(collection)[0])
success, rootless_staging_dir = (
@@ -548,11 +592,15 @@ def _get_representations(self, instance, exp_files):
" This may cause issues on farm."
).format(staging))
+ frame_start = int(instance.get("frameStartHandle"))
+ if instance.get("slate"):
+ frame_start -= 1
+
rep = {
"name": ext,
"ext": ext,
"files": [os.path.basename(f) for f in list(collection)],
- "frameStart": int(instance.get("frameStartHandle")),
+ "frameStart": frame_start,
"frameEnd": int(instance.get("frameEndHandle")),
# If expectedFile are absolute, we need only filenames
"stagingDir": staging,
@@ -598,27 +646,33 @@ def _get_representations(self, instance, exp_files):
"files": os.path.basename(remainder),
"stagingDir": os.path.dirname(remainder),
}
- if "render" in instance.get("families"):
+
+ preview = match_aov_pattern(
+ host_name, self.aov_filter, remainder
+ )
+ if preview:
rep.update({
"fps": instance.get("fps"),
"tags": ["review"]
})
- self._solve_families(instance, True)
+ self._solve_families(instance, preview)
- if (bake_renders
- and remainder in bake_renders[0]["bakeRenderPath"]):
- rep.update({
- "fps": instance.get("fps"),
- "tags": ["review", "delete"]
- })
- # solve families with `preview` attributes
- self._solve_families(instance, True)
- representations.append(rep)
+ already_there = False
+ for repre in instance.get("representations", []):
+ # might be added explicitly before by publish_on_farm
+ already_there = repre.get("files") == rep["files"]
+ if already_there:
+ self.log.debug("repre {} already_there".format(repre))
+ break
+
+ if not already_there:
+ representations.append(rep)
return representations
def _solve_families(self, instance, preview=False):
families = instance.get("families")
+
# if we have one representation with preview tag
# flag whole instance for review and for ftrack
if preview:
@@ -646,10 +700,7 @@ def process(self, instance):
self.context = context
self.anatomy = instance.context.data["anatomy"]
- if hasattr(instance, "_log"):
- data['_log'] = instance._log
-
- asset = data.get("asset") or api.Session["AVALON_ASSET"]
+ asset = data.get("asset") or legacy_io.Session["AVALON_ASSET"]
subset = data.get("subset")
start = instance.data.get("frameStart")
@@ -698,10 +749,17 @@ def process(self, instance):
" This may cause issues."
).format(source))
- families = ["render"]
+ family = "render"
+ if "prerender" in instance.data["families"]:
+ family = "prerender"
+ families = [family]
+
+ # pass review to families if marked as review
+ if data.get("review"):
+ families.append("review")
instance_skeleton_data = {
- "family": "render",
+ "family": family,
"subset": subset,
"families": families,
"asset": asset,
@@ -719,14 +777,12 @@ def process(self, instance):
"resolutionWidth": data.get("resolutionWidth", 1920),
"resolutionHeight": data.get("resolutionHeight", 1080),
"multipartExr": data.get("multipartExr", False),
- "jobBatchName": data.get("jobBatchName", "")
+ "jobBatchName": data.get("jobBatchName", ""),
+ "useSequenceForReview": data.get("useSequenceForReview", True),
+ # map inputVersions `ObjectId` -> `str` so json supports it
+ "inputVersions": list(map(str, data.get("inputVersions", [])))
}
- if "prerender" in instance.data["families"]:
- instance_skeleton_data.update({
- "family": "prerender",
- "families": []})
-
# skip locking version if we are creating v01
instance_version = instance.data.get("version") # take this if exists
if instance_version != 1:
@@ -861,8 +917,10 @@ def process(self, instance):
new_i = copy(i)
new_i["version"] = at.get("version")
new_i["subset"] = at.get("subset")
+ new_i["family"] = at.get("family")
new_i["append"] = True
- new_i["families"].append(at.get("family"))
+ # don't set subsetGroup if we are attaching
+ new_i.pop("subsetGroup")
new_instances.append(new_i)
self.log.info(" - {} / v{}".format(
at.get("subset"), at.get("version")))
@@ -911,12 +969,6 @@ def process(self, instance):
# User is deadline user
render_job["Props"]["User"] = context.data.get(
"deadlineUser", getpass.getuser())
- # Priority is now not handled at all
-
- if self.deadline_priority:
- render_job["Props"]["Pri"] = self.deadline_priority
- else:
- render_job["Props"]["Pri"] = instance.data.get("priority")
render_job["Props"]["Env"] = {
"FTRACK_API_USER": os.environ.get("FTRACK_API_USER"),
@@ -946,7 +998,7 @@ def process(self, instance):
"intent": context.data.get("intent"),
"comment": context.data.get("comment"),
"job": render_job or None,
- "session": api.Session.copy(),
+ "session": legacy_io.Session.copy(),
"instances": instances
}
@@ -990,9 +1042,12 @@ def _extend_frames(self, asset, subset, start, end):
prev_start = None
prev_end = None
- version = openpype.api.get_latest_version(asset_name=asset,
- subset_name=subset
- )
+ project_name = legacy_io.active_project()
+ version = get_last_version_by_subset_name(
+ project_name,
+ subset,
+ asset_name=asset
+ )
# Set prev start / end frames for comparison
if not prev_start and not prev_end:
@@ -1022,7 +1077,7 @@ def _get_publish_folder(self, anatomy, template_data,
get publish_path
Args:
- anatomy (pype.lib.anatomy.Anatomy):
+ anatomy (openpype.pipeline.anatomy.Anatomy):
template_data (dict): pre-calculated collected data for process
asset (string): asset name
subset (string): subset name (actually group name of subset)
@@ -1037,7 +1092,12 @@ def _get_publish_folder(self, anatomy, template_data,
based on 'publish' template
"""
if not version:
- version = openpype.api.get_latest_version(asset, subset)
+ project_name = legacy_io.active_project()
+ version = get_last_version_by_subset_name(
+ project_name,
+ subset,
+ asset_name=asset
+ )
if version:
version = int(version["name"]) + 1
else:
@@ -1054,7 +1114,7 @@ def _get_publish_folder(self, anatomy, template_data,
else:
# solve deprecated situation when `folder` key is not underneath
# `publish` anatomy
- project_name = api.Session["AVALON_PROJECT"]
+ project_name = legacy_io.Session["AVALON_PROJECT"]
self.log.warning((
"Deprecation warning: Anatomy does not have set `folder`"
" key underneath `publish` (in global of for project `{}`)."
diff --git a/openpype/modules/default_modules/deadline/plugins/publish/validate_deadline_connection.py b/openpype/modules/deadline/plugins/publish/validate_deadline_connection.py
similarity index 100%
rename from openpype/modules/default_modules/deadline/plugins/publish/validate_deadline_connection.py
rename to openpype/modules/deadline/plugins/publish/validate_deadline_connection.py
diff --git a/openpype/modules/deadline/plugins/publish/validate_deadline_pools.py b/openpype/modules/deadline/plugins/publish/validate_deadline_pools.py
new file mode 100644
index 00000000000..78eed17c986
--- /dev/null
+++ b/openpype/modules/deadline/plugins/publish/validate_deadline_pools.py
@@ -0,0 +1,48 @@
+import pyblish.api
+
+from openpype.pipeline import (
+ PublishXmlValidationError,
+ OptionalPyblishPluginMixin
+)
+from openpype.modules.deadline.deadline_module import DeadlineModule
+
+
+class ValidateDeadlinePools(OptionalPyblishPluginMixin,
+ pyblish.api.InstancePlugin):
+ """Validate primaryPool and secondaryPool on instance.
+
+ Values are on instance based on value insertion when Creating instance or
+ by Settings in CollectDeadlinePools.
+ """
+
+ label = "Validate Deadline Pools"
+ order = pyblish.api.ValidatorOrder
+ families = ["rendering", "render.farm", "renderFarm", "renderlayer"]
+ optional = True
+
+ def process(self, instance):
+ # get default deadline webservice url from deadline module
+ deadline_url = instance.context.data["defaultDeadline"]
+ self.log.info("deadline_url::{}".format(deadline_url))
+ pools = DeadlineModule.get_deadline_pools(deadline_url, log=self.log)
+ self.log.info("pools::{}".format(pools))
+
+ formatting_data = {
+ "pools_str": ",".join(pools)
+ }
+
+ primary_pool = instance.data.get("primaryPool")
+ if primary_pool and primary_pool not in pools:
+ msg = "Configured primary '{}' not present on Deadline".format(
+ instance.data["primaryPool"])
+ formatting_data["invalid_value_str"] = msg
+ raise PublishXmlValidationError(self, msg,
+ formatting_data=formatting_data)
+
+ secondary_pool = instance.data.get("secondaryPool")
+ if secondary_pool and secondary_pool not in pools:
+ msg = "Configured secondary '{}' not present on Deadline".format(
+ instance.data["secondaryPool"])
+ formatting_data["invalid_value_str"] = msg
+ raise PublishXmlValidationError(self, msg,
+ formatting_data=formatting_data)
diff --git a/openpype/modules/deadline/plugins/publish/validate_expected_and_rendered_files.py b/openpype/modules/deadline/plugins/publish/validate_expected_and_rendered_files.py
new file mode 100644
index 00000000000..f0a3ddd2463
--- /dev/null
+++ b/openpype/modules/deadline/plugins/publish/validate_expected_and_rendered_files.py
@@ -0,0 +1,209 @@
+import os
+import requests
+
+import pyblish.api
+
+from openpype.lib import collect_frames
+from openpype_modules.deadline.abstract_submit_deadline import requests_get
+
+
+class ValidateExpectedFiles(pyblish.api.InstancePlugin):
+ """Compare rendered and expected files"""
+
+ label = "Validate rendered files from Deadline"
+ order = pyblish.api.ValidatorOrder
+ families = ["render"]
+ targets = ["deadline"]
+
+ # check if actual frame range on render job wasn't different
+ # case when artists wants to render only subset of frames
+ allow_user_override = True
+
+ def process(self, instance):
+ self.instance = instance
+ frame_list = self._get_frame_list(instance.data["render_job_id"])
+
+ for repre in instance.data["representations"]:
+ expected_files = self._get_expected_files(repre)
+
+ staging_dir = repre["stagingDir"]
+ existing_files = self._get_existing_files(staging_dir)
+
+ if self.allow_user_override:
+ # We always check for user override because the user might have
+ # also overridden the Job frame list to be longer than the
+ # originally submitted frame range
+ # todo: We should first check if Job frame range was overridden
+ # at all so we don't unnecessarily override anything
+ file_name_template, frame_placeholder = \
+ self._get_file_name_template_and_placeholder(
+ expected_files)
+
+ if not file_name_template:
+ raise RuntimeError("Unable to retrieve file_name template"
+ "from files: {}".format(expected_files))
+
+ job_expected_files = self._get_job_expected_files(
+ file_name_template,
+ frame_placeholder,
+ frame_list)
+
+ job_files_diff = job_expected_files.difference(expected_files)
+ if job_files_diff:
+ self.log.debug(
+ "Detected difference in expected output files from "
+ "Deadline job. Assuming an updated frame list by the "
+ "user. Difference: {}".format(sorted(job_files_diff))
+ )
+
+ # Update the representation expected files
+ self.log.info("Update range from actual job range "
+ "to frame list: {}".format(frame_list))
+ repre["files"] = sorted(job_expected_files)
+
+ # Update the expected files
+ expected_files = job_expected_files
+
+ # We don't use set.difference because we do allow other existing
+ # files to be in the folder that we might not want to use.
+ missing = expected_files - existing_files
+ if missing:
+ raise RuntimeError("Missing expected files: {}".format(
+ sorted(missing)))
+
+ def _get_frame_list(self, original_job_id):
+ """Returns list of frame ranges from all render job.
+
+ Render job might be re-submitted so job_id in metadata.json could be
+ invalid. GlobalJobPreload injects current job id to RENDER_JOB_IDS.
+
+ Args:
+ original_job_id (str)
+ Returns:
+ (list)
+ """
+ all_frame_lists = []
+ render_job_ids = os.environ.get("RENDER_JOB_IDS")
+ if render_job_ids:
+ render_job_ids = render_job_ids.split(',')
+ else: # fallback
+ render_job_ids = [original_job_id]
+
+ for job_id in render_job_ids:
+ job_info = self._get_job_info(job_id)
+ frame_list = job_info["Props"]["Frames"]
+ if frame_list:
+ all_frame_lists.extend(frame_list.split(','))
+
+ return all_frame_lists
+
+ def _get_job_expected_files(self,
+ file_name_template,
+ frame_placeholder,
+ frame_list):
+ """Calculates list of names of expected rendered files.
+
+ Might be different from expected files from submission if user
+ explicitly and manually changed the frame list on the Deadline job.
+
+ """
+ # no frames in file name at all, eg 'renderCompositingMain.withLut.mov'
+ if not frame_placeholder:
+ return set([file_name_template])
+
+ real_expected_rendered = set()
+ src_padding_exp = "%0{}d".format(len(frame_placeholder))
+ for frames in frame_list:
+ if '-' not in frames: # single frame
+ frames = "{}-{}".format(frames, frames)
+
+ start, end = frames.split('-')
+ for frame in range(int(start), int(end) + 1):
+ ren_name = file_name_template.replace(
+ frame_placeholder, src_padding_exp % frame)
+ real_expected_rendered.add(ren_name)
+
+ return real_expected_rendered
+
+ def _get_file_name_template_and_placeholder(self, files):
+ """Returns file name with frame replaced with # and this placeholder"""
+ sources_and_frames = collect_frames(files)
+
+ file_name_template = frame_placeholder = None
+ for file_name, frame in sources_and_frames.items():
+
+ # There might be cases where clique was unable to collect
+ # collections in `collect_frames` - thus we capture that case
+ if frame is not None:
+ frame_placeholder = "#" * len(frame)
+
+ file_name_template = os.path.basename(
+ file_name.replace(frame, frame_placeholder))
+ else:
+ file_name_template = file_name
+ break
+
+ return file_name_template, frame_placeholder
+
+ def _get_job_info(self, job_id):
+ """Calls DL for actual job info for 'job_id'
+
+ Might be different than job info saved in metadata.json if user
+ manually changes job pre/during rendering.
+
+ """
+ # get default deadline webservice url from deadline module
+ deadline_url = self.instance.context.data["defaultDeadline"]
+ # if custom one is set in instance, use that
+ if self.instance.data.get("deadlineUrl"):
+ deadline_url = self.instance.data.get("deadlineUrl")
+ assert deadline_url, "Requires Deadline Webservice URL"
+
+ url = "{}/api/jobs?JobID={}".format(deadline_url, job_id)
+ try:
+ response = requests_get(url)
+ except requests.exceptions.ConnectionError:
+ self.log.error("Deadline is not accessible at "
+ "{}".format(deadline_url))
+ return {}
+
+ if not response.ok:
+ self.log.error("Submission failed!")
+ self.log.error(response.status_code)
+ self.log.error(response.content)
+ raise RuntimeError(response.text)
+
+ json_content = response.json()
+ if json_content:
+ return json_content.pop()
+ return {}
+
+ def _get_existing_files(self, staging_dir):
+ """Returns set of existing file names from 'staging_dir'"""
+ existing_files = set()
+ for file_name in os.listdir(staging_dir):
+ existing_files.add(file_name)
+ return existing_files
+
+ def _get_expected_files(self, repre):
+ """Returns set of file names in representation['files']
+
+ The representations are collected from `CollectRenderedFiles` using
+ the metadata.json file submitted along with the render job.
+
+ Args:
+ repre (dict): The representation containing 'files'
+
+ Returns:
+ set: Set of expected file_names in the staging directory.
+
+ """
+ expected_files = set()
+
+ files = repre["files"]
+ if not isinstance(files, list):
+ files = [files]
+
+ for file_name in files:
+ expected_files.add(file_name)
+ return expected_files
diff --git a/openpype/modules/deadline/repository/custom/plugins/GlobalJobPreLoad.py b/openpype/modules/deadline/repository/custom/plugins/GlobalJobPreLoad.py
new file mode 100644
index 00000000000..61b95cf06da
--- /dev/null
+++ b/openpype/modules/deadline/repository/custom/plugins/GlobalJobPreLoad.py
@@ -0,0 +1,264 @@
+# -*- coding: utf-8 -*-
+import os
+import tempfile
+from datetime import datetime
+import subprocess
+import json
+import platform
+import uuid
+import re
+from Deadline.Scripting import RepositoryUtils, FileUtils, DirectoryUtils
+
+
+def get_openpype_version_from_path(path, build=True):
+ """Get OpenPype version from provided path.
+ path (str): Path to scan.
+ build (bool, optional): Get only builds, not sources
+
+ Returns:
+ str or None: version of OpenPype if found.
+
+ """
+ # fix path for application bundle on macos
+ if platform.system().lower() == "darwin":
+ path = os.path.join(path, "Contents", "MacOS", "lib", "Python")
+
+ version_file = os.path.join(path, "openpype", "version.py")
+ if not os.path.isfile(version_file):
+ return None
+
+ # skip if the version is not build
+ exe = os.path.join(path, "openpype_console.exe")
+ if platform.system().lower() in ["linux", "darwin"]:
+ exe = os.path.join(path, "openpype_console")
+
+ # if only builds are requested
+ if build and not os.path.isfile(exe): # noqa: E501
+ print(" ! path is not a build: {}".format(path))
+ return None
+
+ version = {}
+ with open(version_file, "r") as vf:
+ exec(vf.read(), version)
+
+ version_match = re.search(r"(\d+\.\d+.\d+).*", version["__version__"])
+ return version_match[1]
+
+
+def get_openpype_executable():
+ """Return OpenPype Executable from Event Plug-in Settings"""
+ config = RepositoryUtils.GetPluginConfig("OpenPype")
+ exe_list = config.GetConfigEntryWithDefault("OpenPypeExecutable", "")
+ dir_list = config.GetConfigEntryWithDefault(
+ "OpenPypeInstallationDirs", "")
+ return exe_list, dir_list
+
+
+def inject_openpype_environment(deadlinePlugin):
+ """ Pull env vars from OpenPype and push them to rendering process.
+
+ Used for correct paths, configuration from OpenPype etc.
+ """
+ job = deadlinePlugin.GetJob()
+
+ print(">>> Injecting OpenPype environments ...")
+ try:
+ print(">>> Getting OpenPype executable ...")
+ exe_list, dir_list = get_openpype_executable()
+ openpype_versions = []
+ # if the job requires specific OpenPype version,
+ # lets go over all available and find compatible build.
+ requested_version = job.GetJobEnvironmentKeyValue("OPENPYPE_VERSION")
+ if requested_version:
+ print((
+ ">>> Scanning for compatible requested version {}"
+ ).format(requested_version))
+ install_dir = DirectoryUtils.SearchDirectoryList(dir_list)
+ if install_dir:
+ print("--- Looking for OpenPype at: {}".format(install_dir))
+ sub_dirs = [
+ f.path for f in os.scandir(install_dir)
+ if f.is_dir()
+ ]
+ for subdir in sub_dirs:
+ version = get_openpype_version_from_path(subdir)
+ if not version:
+ continue
+ print(" - found: {} - {}".format(version, subdir))
+ openpype_versions.append((version, subdir))
+
+ exe = FileUtils.SearchFileList(exe_list)
+ if openpype_versions:
+ # if looking for requested compatible version,
+ # add the implicitly specified to the list too.
+ print("Looking for OpenPype at: {}".format(os.path.dirname(exe)))
+ version = get_openpype_version_from_path(
+ os.path.dirname(exe))
+ if version:
+ print(" - found: {} - {}".format(
+ version, os.path.dirname(exe)
+ ))
+ openpype_versions.append((version, os.path.dirname(exe)))
+
+ if requested_version:
+ # sort detected versions
+ if openpype_versions:
+ # use natural sorting
+ openpype_versions.sort(
+ key=lambda ver: [
+ int(t) if t.isdigit() else t.lower()
+ for t in re.split(r"(\d+)", ver[0])
+ ])
+ print((
+ "*** Latest available version found is {}"
+ ).format(openpype_versions[-1][0]))
+ requested_major, requested_minor, _ = requested_version.split(".")[:3] # noqa: E501
+ compatible_versions = []
+ for version in openpype_versions:
+ v = version[0].split(".")[:3]
+ if v[0] == requested_major and v[1] == requested_minor:
+ compatible_versions.append(version)
+ if not compatible_versions:
+ raise RuntimeError(
+ ("Cannot find compatible version available "
+ "for version {} requested by the job. "
+ "Please add it through plugin configuration "
+ "in Deadline or install it to configured "
+ "directory.").format(requested_version))
+ # sort compatible versions nad pick the last one
+ compatible_versions.sort(
+ key=lambda ver: [
+ int(t) if t.isdigit() else t.lower()
+ for t in re.split(r"(\d+)", ver[0])
+ ])
+ print((
+ "*** Latest compatible version found is {}"
+ ).format(compatible_versions[-1][0]))
+ # create list of executables for different platform and let
+ # Deadline decide.
+ exe_list = [
+ os.path.join(
+ compatible_versions[-1][1], "openpype_console.exe"),
+ os.path.join(
+ compatible_versions[-1][1], "openpype_console")
+ ]
+ exe = FileUtils.SearchFileList(";".join(exe_list))
+ if exe == "":
+ raise RuntimeError(
+ "OpenPype executable was not found " +
+ "in the semicolon separated list " +
+ "\"" + ";".join(exe_list) + "\". " +
+ "The path to the render executable can be configured " +
+ "from the Plugin Configuration in the Deadline Monitor.")
+
+ print("--- OpenPype executable: {}".format(exe))
+
+ # tempfile.TemporaryFile cannot be used because of locking
+ temp_file_name = "{}_{}.json".format(
+ datetime.utcnow().strftime('%Y%m%d%H%M%S%f'),
+ str(uuid.uuid1())
+ )
+ export_url = os.path.join(tempfile.gettempdir(), temp_file_name)
+ print(">>> Temporary path: {}".format(export_url))
+
+ args = [
+ exe,
+ "--headless",
+ 'extractenvironments',
+ export_url
+ ]
+
+ add_args = {}
+ add_args['project'] = \
+ job.GetJobEnvironmentKeyValue('AVALON_PROJECT')
+ add_args['asset'] = job.GetJobEnvironmentKeyValue('AVALON_ASSET')
+ add_args['task'] = job.GetJobEnvironmentKeyValue('AVALON_TASK')
+ add_args['app'] = job.GetJobEnvironmentKeyValue('AVALON_APP_NAME')
+ add_args["envgroup"] = "farm"
+
+ if all(add_args.values()):
+ for key, value in add_args.items():
+ args.append("--{}".format(key))
+ args.append(value)
+ else:
+ msg = "Required env vars: AVALON_PROJECT, AVALON_ASSET, " + \
+ "AVALON_TASK, AVALON_APP_NAME"
+ raise RuntimeError(msg)
+
+ if not os.environ.get("OPENPYPE_MONGO"):
+ print(">>> Missing OPENPYPE_MONGO env var, process won't work")
+
+ env = os.environ
+ env["OPENPYPE_HEADLESS_MODE"] = "1"
+ env["AVALON_TIMEOUT"] = "5000"
+
+ print(">>> Executing: {}".format(" ".join(args)))
+ std_output = subprocess.check_output(args,
+ cwd=os.path.dirname(exe),
+ env=env)
+ print(">>> Process result {}".format(std_output))
+
+ print(">>> Loading file ...")
+ with open(export_url) as fp:
+ contents = json.load(fp)
+ for key, value in contents.items():
+ deadlinePlugin.SetProcessEnvironmentVariable(key, value)
+
+ script_url = job.GetJobPluginInfoKeyValue("ScriptFilename")
+ if script_url:
+
+ script_url = script_url.format(**contents).replace("\\", "/")
+ print(">>> Setting script path {}".format(script_url))
+ job.SetJobPluginInfoKeyValue("ScriptFilename", script_url)
+
+ print(">>> Removing temporary file")
+ os.remove(export_url)
+
+ print(">> Injection end.")
+ except Exception as e:
+ if hasattr(e, "output"):
+ print(">>> Exception {}".format(e.output))
+ import traceback
+ print(traceback.format_exc())
+ print("!!! Injection failed.")
+ RepositoryUtils.FailJob(job)
+ raise
+
+
+def inject_render_job_id(deadlinePlugin):
+ """Inject dependency ids to publish process as env var for validation."""
+ print(">>> Injecting render job id ...")
+ job = deadlinePlugin.GetJob()
+
+ dependency_ids = job.JobDependencyIDs
+ print(">>> Dependency IDs: {}".format(dependency_ids))
+ render_job_ids = ",".join(dependency_ids)
+
+ deadlinePlugin.SetProcessEnvironmentVariable("RENDER_JOB_IDS",
+ render_job_ids)
+ print(">>> Injection end.")
+
+
+def __main__(deadlinePlugin):
+ print("*** GlobalJobPreload start ...")
+ print(">>> Getting job ...")
+ job = deadlinePlugin.GetJob()
+
+ openpype_render_job = \
+ job.GetJobEnvironmentKeyValue('OPENPYPE_RENDER_JOB') or '0'
+ openpype_publish_job = \
+ job.GetJobEnvironmentKeyValue('OPENPYPE_PUBLISH_JOB') or '0'
+ openpype_remote_job = \
+ job.GetJobEnvironmentKeyValue('OPENPYPE_REMOTE_JOB') or '0'
+
+ print("--- Job type - render {}".format(openpype_render_job))
+ print("--- Job type - publish {}".format(openpype_publish_job))
+ print("--- Job type - remote {}".format(openpype_remote_job))
+ if openpype_publish_job == '1' and openpype_render_job == '1':
+ raise RuntimeError("Misconfiguration. Job couldn't be both " +
+ "render and publish.")
+
+ if openpype_publish_job == '1':
+ inject_render_job_id(deadlinePlugin)
+ elif openpype_render_job == '1' or openpype_remote_job == '1':
+ inject_openpype_environment(deadlinePlugin)
diff --git a/vendor/deadline/custom/plugins/HarmonyOpenPype/HarmonyOpenPype.ico b/openpype/modules/deadline/repository/custom/plugins/HarmonyOpenPype/HarmonyOpenPype.ico
similarity index 100%
rename from vendor/deadline/custom/plugins/HarmonyOpenPype/HarmonyOpenPype.ico
rename to openpype/modules/deadline/repository/custom/plugins/HarmonyOpenPype/HarmonyOpenPype.ico
diff --git a/vendor/deadline/custom/plugins/HarmonyOpenPype/HarmonyOpenPype.options b/openpype/modules/deadline/repository/custom/plugins/HarmonyOpenPype/HarmonyOpenPype.options
similarity index 100%
rename from vendor/deadline/custom/plugins/HarmonyOpenPype/HarmonyOpenPype.options
rename to openpype/modules/deadline/repository/custom/plugins/HarmonyOpenPype/HarmonyOpenPype.options
diff --git a/vendor/deadline/custom/plugins/HarmonyOpenPype/HarmonyOpenPype.param b/openpype/modules/deadline/repository/custom/plugins/HarmonyOpenPype/HarmonyOpenPype.param
similarity index 100%
rename from vendor/deadline/custom/plugins/HarmonyOpenPype/HarmonyOpenPype.param
rename to openpype/modules/deadline/repository/custom/plugins/HarmonyOpenPype/HarmonyOpenPype.param
diff --git a/vendor/deadline/custom/plugins/HarmonyOpenPype/HarmonyOpenPype.py b/openpype/modules/deadline/repository/custom/plugins/HarmonyOpenPype/HarmonyOpenPype.py
similarity index 100%
rename from vendor/deadline/custom/plugins/HarmonyOpenPype/HarmonyOpenPype.py
rename to openpype/modules/deadline/repository/custom/plugins/HarmonyOpenPype/HarmonyOpenPype.py
diff --git a/vendor/deadline/custom/plugins/OpenPype/OpenPype.ico b/openpype/modules/deadline/repository/custom/plugins/OpenPype/OpenPype.ico
similarity index 100%
rename from vendor/deadline/custom/plugins/OpenPype/OpenPype.ico
rename to openpype/modules/deadline/repository/custom/plugins/OpenPype/OpenPype.ico
diff --git a/vendor/deadline/custom/plugins/OpenPype/OpenPype.options b/openpype/modules/deadline/repository/custom/plugins/OpenPype/OpenPype.options
similarity index 100%
rename from vendor/deadline/custom/plugins/OpenPype/OpenPype.options
rename to openpype/modules/deadline/repository/custom/plugins/OpenPype/OpenPype.options
diff --git a/openpype/modules/deadline/repository/custom/plugins/OpenPype/OpenPype.param b/openpype/modules/deadline/repository/custom/plugins/OpenPype/OpenPype.param
new file mode 100644
index 00000000000..b3ac18e20ca
--- /dev/null
+++ b/openpype/modules/deadline/repository/custom/plugins/OpenPype/OpenPype.param
@@ -0,0 +1,27 @@
+[About]
+Type=label
+Label=About
+Category=About Plugin
+CategoryOrder=-1
+Index=0
+Default=OpenPype Plugin for Deadline
+Description=Not configurable
+
+[OpenPypeInstallationDirs]
+Type=multilinemultifolder
+Label=Directories where OpenPype versions are installed
+Category=OpenPype Installation Directories
+CategoryOrder=0
+Index=0
+Default=C:\Program Files (x86)\OpenPype
+Description=Path or paths to directories where multiple versions of OpenPype might be installed. Enter every such path on separate lines.
+
+[OpenPypeExecutable]
+Type=multilinemultifilename
+Label=OpenPype Executable
+Category=OpenPype Executables
+CategoryOrder=1
+Index=0
+Default=
+Description=The path to the OpenPype executable. Enter alternative paths on separate lines.
+
diff --git a/openpype/modules/deadline/repository/custom/plugins/OpenPype/OpenPype.py b/openpype/modules/deadline/repository/custom/plugins/OpenPype/OpenPype.py
new file mode 100644
index 00000000000..6b0f69d98f0
--- /dev/null
+++ b/openpype/modules/deadline/repository/custom/plugins/OpenPype/OpenPype.py
@@ -0,0 +1,224 @@
+#!/usr/bin/env python3
+
+from System.IO import Path
+from System.Text.RegularExpressions import Regex
+
+from Deadline.Plugins import PluginType, DeadlinePlugin
+from Deadline.Scripting import (
+ StringUtils,
+ FileUtils,
+ DirectoryUtils,
+ RepositoryUtils
+)
+
+import re
+import os
+import platform
+
+
+######################################################################
+# This is the function that Deadline calls to get an instance of the
+# main DeadlinePlugin class.
+######################################################################
+def GetDeadlinePlugin():
+ return OpenPypeDeadlinePlugin()
+
+
+def CleanupDeadlinePlugin(deadlinePlugin):
+ deadlinePlugin.Cleanup()
+
+
+class OpenPypeDeadlinePlugin(DeadlinePlugin):
+ """
+ Standalone plugin for publishing from OpenPype.
+
+ Calls OpenPype executable 'openpype_console' from first correctly found
+ file based on plugin configuration. Uses 'publish' command and passes
+ path to metadata json file, which contains all needed information
+ for publish process.
+ """
+ def __init__(self):
+ self.InitializeProcessCallback += self.InitializeProcess
+ self.RenderExecutableCallback += self.RenderExecutable
+ self.RenderArgumentCallback += self.RenderArgument
+
+ def Cleanup(self):
+ for stdoutHandler in self.StdoutHandlers:
+ del stdoutHandler.HandleCallback
+
+ del self.InitializeProcessCallback
+ del self.RenderExecutableCallback
+ del self.RenderArgumentCallback
+
+ def InitializeProcess(self):
+ self.PluginType = PluginType.Simple
+ self.StdoutHandling = True
+
+ self.SingleFramesOnly = self.GetBooleanPluginInfoEntryWithDefault(
+ "SingleFramesOnly", False)
+ self.LogInfo("Single Frames Only: %s" % self.SingleFramesOnly)
+
+ self.AddStdoutHandlerCallback(
+ ".*Progress: (\d+)%.*").HandleCallback += self.HandleProgress
+
+ @staticmethod
+ def get_openpype_version_from_path(path, build=True):
+ """Get OpenPype version from provided path.
+ path (str): Path to scan.
+ build (bool, optional): Get only builds, not sources
+
+ Returns:
+ str or None: version of OpenPype if found.
+
+ """
+ # fix path for application bundle on macos
+ if platform.system().lower() == "darwin":
+ path = os.path.join(path, "Contents", "MacOS", "lib", "Python")
+
+ version_file = os.path.join(path, "openpype", "version.py")
+ if not os.path.isfile(version_file):
+ return None
+
+ # skip if the version is not build
+ exe = os.path.join(path, "openpype_console.exe")
+ if platform.system().lower() in ["linux", "darwin"]:
+ exe = os.path.join(path, "openpype_console")
+
+ # if only builds are requested
+ if build and not os.path.isfile(exe): # noqa: E501
+ print(f" ! path is not a build: {path}")
+ return None
+
+ version = {}
+ with open(version_file, "r") as vf:
+ exec(vf.read(), version)
+
+ version_match = re.search(r"(\d+\.\d+.\d+).*", version["__version__"])
+ return version_match[1]
+
+ def RenderExecutable(self):
+ job = self.GetJob()
+ openpype_versions = []
+ # if the job requires specific OpenPype version,
+ # lets go over all available and find compatible build.
+ requested_version = job.GetJobEnvironmentKeyValue("OPENPYPE_VERSION")
+ if requested_version:
+ self.LogInfo((
+ "Scanning for compatible requested "
+ f"version {requested_version}"))
+ dir_list = self.GetConfigEntry("OpenPypeInstallationDirs")
+ install_dir = DirectoryUtils.SearchDirectoryList(dir_list)
+ if dir:
+ sub_dirs = [
+ f.path for f in os.scandir(install_dir)
+ if f.is_dir()
+ ]
+ for subdir in sub_dirs:
+ version = self.get_openpype_version_from_path(subdir)
+ if not version:
+ continue
+ openpype_versions.append((version, subdir))
+
+ exe_list = self.GetConfigEntry("OpenPypeExecutable")
+ exe = FileUtils.SearchFileList(exe_list)
+ if openpype_versions:
+ # if looking for requested compatible version,
+ # add the implicitly specified to the list too.
+ version = self.get_openpype_version_from_path(
+ os.path.dirname(exe))
+ if version:
+ openpype_versions.append((version, os.path.dirname(exe)))
+
+ if requested_version:
+ # sort detected versions
+ if openpype_versions:
+ openpype_versions.sort(
+ key=lambda ver: [
+ int(t) if t.isdigit() else t.lower()
+ for t in re.split(r"(\d+)", ver[0])
+ ])
+ requested_major, requested_minor, _ = requested_version.split(".")[:3] # noqa: E501
+ compatible_versions = []
+ for version in openpype_versions:
+ v = version[0].split(".")[:3]
+ if v[0] == requested_major and v[1] == requested_minor:
+ compatible_versions.append(version)
+ if not compatible_versions:
+ self.FailRender(("Cannot find compatible version available "
+ "for version {} requested by the job. "
+ "Please add it through plugin configuration "
+ "in Deadline or install it to configured "
+ "directory.").format(requested_version))
+ # sort compatible versions nad pick the last one
+ compatible_versions.sort(
+ key=lambda ver: [
+ int(t) if t.isdigit() else t.lower()
+ for t in re.split(r"(\d+)", ver[0])
+ ])
+ # create list of executables for different platform and let
+ # Deadline decide.
+ exe_list = [
+ os.path.join(
+ compatible_versions[-1][1], "openpype_console.exe"),
+ os.path.join(
+ compatible_versions[-1][1], "openpype_console")
+ ]
+ exe = FileUtils.SearchFileList(";".join(exe_list))
+
+ if exe == "":
+ self.FailRender(
+ "OpenPype executable was not found " +
+ "in the semicolon separated list " +
+ "\"" + ";".join(exe_list) + "\". " +
+ "The path to the render executable can be configured " +
+ "from the Plugin Configuration in the Deadline Monitor.")
+ return exe
+
+ def RenderArgument(self):
+ arguments = str(self.GetPluginInfoEntryWithDefault("Arguments", ""))
+ arguments = RepositoryUtils.CheckPathMapping(arguments)
+
+ arguments = re.sub(r"<(?i)STARTFRAME>", str(self.GetStartFrame()),
+ arguments)
+ arguments = re.sub(r"<(?i)ENDFRAME>", str(self.GetEndFrame()),
+ arguments)
+ arguments = re.sub(r"<(?i)QUOTE>", "\"", arguments)
+
+ arguments = self.ReplacePaddedFrame(arguments,
+ "<(?i)STARTFRAME%([0-9]+)>",
+ self.GetStartFrame())
+ arguments = self.ReplacePaddedFrame(arguments,
+ "<(?i)ENDFRAME%([0-9]+)>",
+ self.GetEndFrame())
+
+ count = 0
+ for filename in self.GetAuxiliaryFilenames():
+ localAuxFile = Path.Combine(self.GetJobsDataDirectory(), filename)
+ arguments = re.sub(r"<(?i)AUXFILE" + str(count) + r">",
+ localAuxFile.replace("\\", "/"), arguments)
+ count += 1
+
+ return arguments
+
+ def ReplacePaddedFrame(self, arguments, pattern, frame):
+ frameRegex = Regex(pattern)
+ while True:
+ frameMatch = frameRegex.Match(arguments)
+ if frameMatch.Success:
+ paddingSize = int(frameMatch.Groups[1].Value)
+ if paddingSize > 0:
+ padding = StringUtils.ToZeroPaddedString(frame,
+ paddingSize,
+ False)
+ else:
+ padding = str(frame)
+ arguments = arguments.replace(frameMatch.Groups[0].Value,
+ padding)
+ else:
+ break
+
+ return arguments
+
+ def HandleProgress(self):
+ progress = float(self.GetRegexMatch(1))
+ self.SetProgress(progress)
diff --git a/vendor/deadline/custom/plugins/OpenPypeTileAssembler/OpenPypeTileAssembler.ico b/openpype/modules/deadline/repository/custom/plugins/OpenPypeTileAssembler/OpenPypeTileAssembler.ico
similarity index 100%
rename from vendor/deadline/custom/plugins/OpenPypeTileAssembler/OpenPypeTileAssembler.ico
rename to openpype/modules/deadline/repository/custom/plugins/OpenPypeTileAssembler/OpenPypeTileAssembler.ico
diff --git a/vendor/deadline/custom/plugins/OpenPypeTileAssembler/OpenPypeTileAssembler.options b/openpype/modules/deadline/repository/custom/plugins/OpenPypeTileAssembler/OpenPypeTileAssembler.options
similarity index 100%
rename from vendor/deadline/custom/plugins/OpenPypeTileAssembler/OpenPypeTileAssembler.options
rename to openpype/modules/deadline/repository/custom/plugins/OpenPypeTileAssembler/OpenPypeTileAssembler.options
diff --git a/vendor/deadline/custom/plugins/OpenPypeTileAssembler/OpenPypeTileAssembler.param b/openpype/modules/deadline/repository/custom/plugins/OpenPypeTileAssembler/OpenPypeTileAssembler.param
similarity index 100%
rename from vendor/deadline/custom/plugins/OpenPypeTileAssembler/OpenPypeTileAssembler.param
rename to openpype/modules/deadline/repository/custom/plugins/OpenPypeTileAssembler/OpenPypeTileAssembler.param
diff --git a/openpype/modules/deadline/repository/custom/plugins/OpenPypeTileAssembler/OpenPypeTileAssembler.py b/openpype/modules/deadline/repository/custom/plugins/OpenPypeTileAssembler/OpenPypeTileAssembler.py
new file mode 100644
index 00000000000..0d356785f97
--- /dev/null
+++ b/openpype/modules/deadline/repository/custom/plugins/OpenPypeTileAssembler/OpenPypeTileAssembler.py
@@ -0,0 +1,533 @@
+# -*- coding: utf-8 -*-
+"""Tile Assembler Plugin using Open Image IO tool.
+
+Todo:
+ Currently we support only EXRs with their data window set.
+"""
+import os
+import re
+import subprocess
+import xml.etree.ElementTree
+
+from System.IO import Path
+
+from Deadline.Plugins import DeadlinePlugin
+from Deadline.Scripting import (
+ FileUtils, RepositoryUtils, SystemUtils)
+
+
+STRING_TAGS = {
+ "format"
+}
+INT_TAGS = {
+ "x", "y", "z",
+ "width", "height", "depth",
+ "full_x", "full_y", "full_z",
+ "full_width", "full_height", "full_depth",
+ "tile_width", "tile_height", "tile_depth",
+ "nchannels",
+ "alpha_channel",
+ "z_channel",
+ "deep",
+ "subimages",
+}
+
+
+XML_CHAR_REF_REGEX_HEX = re.compile(r"&#x?[0-9a-fA-F]+;")
+
+# Regex to parse array attributes
+ARRAY_TYPE_REGEX = re.compile(r"^(int|float|string)\[\d+\]$")
+
+
+def convert_value_by_type_name(value_type, value):
+ """Convert value to proper type based on type name.
+
+ In some cases value types have custom python class.
+ """
+
+ # Simple types
+ if value_type == "string":
+ return value
+
+ if value_type == "int":
+ return int(value)
+
+ if value_type == "float":
+ return float(value)
+
+ # Vectors will probably have more types
+ if value_type == "vec2f":
+ return [float(item) for item in value.split(",")]
+
+ # Matrix should be always have square size of element 3x3, 4x4
+ # - are returned as list of lists
+ if value_type == "matrix":
+ output = []
+ current_index = -1
+ parts = value.split(",")
+ parts_len = len(parts)
+ if parts_len == 1:
+ divisor = 1
+ elif parts_len == 4:
+ divisor = 2
+ elif parts_len == 9:
+ divisor = 3
+ elif parts_len == 16:
+ divisor = 4
+ else:
+ print("Unknown matrix resolution {}. Value: \"{}\"".format(
+ parts_len, value
+ ))
+ for part in parts:
+ output.append(float(part))
+ return output
+
+ for idx, item in enumerate(parts):
+ list_index = idx % divisor
+ if list_index > current_index:
+ current_index = list_index
+ output.append([])
+ output[list_index].append(float(item))
+ return output
+
+ if value_type == "rational2i":
+ parts = value.split("/")
+ top = float(parts[0])
+ bottom = 1.0
+ if len(parts) != 1:
+ bottom = float(parts[1])
+ return float(top) / float(bottom)
+
+ if value_type == "vector":
+ parts = [part.strip() for part in value.split(",")]
+ output = []
+ for part in parts:
+ if part == "-nan":
+ output.append(None)
+ continue
+ try:
+ part = float(part)
+ except ValueError:
+ pass
+ output.append(part)
+ return output
+
+ if value_type == "timecode":
+ return value
+
+ # Array of other types is converted to list
+ re_result = ARRAY_TYPE_REGEX.findall(value_type)
+ if re_result:
+ array_type = re_result[0]
+ output = []
+ for item in value.split(","):
+ output.append(
+ convert_value_by_type_name(array_type, item)
+ )
+ return output
+
+ print((
+ "MISSING IMPLEMENTATION:"
+ " Unknown attrib type \"{}\". Value: {}"
+ ).format(value_type, value))
+ return value
+
+
+def parse_oiio_xml_output(xml_string):
+ """Parse xml output from OIIO info command."""
+ output = {}
+ if not xml_string:
+ return output
+
+ # Fix values with ampresand (lazy fix)
+ # - oiiotool exports invalid xml which ElementTree can't handle
+ # e.g. ""
+ # WARNING: this will affect even valid character entities. If you need
+ # those values correctly, this must take care of valid character ranges.
+ # See https://github.com/pypeclub/OpenPype/pull/2729
+ matches = XML_CHAR_REF_REGEX_HEX.findall(xml_string)
+ for match in matches:
+ new_value = match.replace("&", "&")
+ xml_string = xml_string.replace(match, new_value)
+
+ tree = xml.etree.ElementTree.fromstring(xml_string)
+ attribs = {}
+ output["attribs"] = attribs
+ for child in tree:
+ tag_name = child.tag
+ if tag_name == "attrib":
+ attrib_def = child.attrib
+ value = convert_value_by_type_name(
+ attrib_def["type"], child.text
+ )
+
+ attribs[attrib_def["name"]] = value
+ continue
+
+ # Channels are stored as tex on each child
+ if tag_name == "channelnames":
+ value = []
+ for channel in child:
+ value.append(channel.text)
+
+ # Convert known integer type tags to int
+ elif tag_name in INT_TAGS:
+ value = int(child.text)
+
+ # Keep value of known string tags
+ elif tag_name in STRING_TAGS:
+ value = child.text
+
+ # Keep value as text for unknown tags
+ # - feel free to add more tags
+ else:
+ value = child.text
+ print((
+ "MISSING IMPLEMENTATION:"
+ " Unknown tag \"{}\". Value \"{}\""
+ ).format(tag_name, value))
+
+ output[child.tag] = value
+
+ return output
+
+
+def info_about_input(oiiotool_path, filepath):
+ args = [
+ oiiotool_path,
+ "--info",
+ "-v",
+ "-i:infoformat=xml",
+ filepath
+ ]
+ popen = subprocess.Popen(args, stdout=subprocess.PIPE)
+ _stdout, _stderr = popen.communicate()
+ output = ""
+ if _stdout:
+ output += _stdout.decode("utf-8")
+
+ if _stderr:
+ output += _stderr.decode("utf-8")
+
+ output = output.replace("\r\n", "\n")
+ xml_started = False
+ lines = []
+ for line in output.split("\n"):
+ if not xml_started:
+ if not line.startswith("<"):
+ continue
+ xml_started = True
+ if xml_started:
+ lines.append(line)
+
+ if not xml_started:
+ raise ValueError(
+ "Failed to read input file \"{}\".\nOutput:\n{}".format(
+ filepath, output
+ )
+ )
+ xml_text = "\n".join(lines)
+ return parse_oiio_xml_output(xml_text)
+
+
+def GetDeadlinePlugin(): # noqa: N802
+ """Helper."""
+ return OpenPypeTileAssembler()
+
+
+def CleanupDeadlinePlugin(deadlinePlugin): # noqa: N802, N803
+ """Helper."""
+ deadlinePlugin.cleanup()
+
+
+class OpenPypeTileAssembler(DeadlinePlugin):
+ """Deadline plugin for assembling tiles using OIIO."""
+
+ def __init__(self):
+ """Init."""
+ self.InitializeProcessCallback += self.initialize_process
+ self.RenderExecutableCallback += self.render_executable
+ self.RenderArgumentCallback += self.render_argument
+ self.PreRenderTasksCallback += self.pre_render_tasks
+ self.PostRenderTasksCallback += self.post_render_tasks
+
+ def cleanup(self):
+ """Cleanup function."""
+ for stdoutHandler in self.StdoutHandlers:
+ del stdoutHandler.HandleCallback
+
+ del self.InitializeProcessCallback
+ del self.RenderExecutableCallback
+ del self.RenderArgumentCallback
+ del self.PreRenderTasksCallback
+ del self.PostRenderTasksCallback
+
+ def initialize_process(self):
+ """Initialization."""
+ self.SingleFramesOnly = True
+ self.StdoutHandling = True
+ self.renderer = self.GetPluginInfoEntryWithDefault(
+ "Renderer", "undefined")
+ self.AddStdoutHandlerCallback(
+ ".*Error.*").HandleCallback += self.handle_stdout_error
+
+ def render_executable(self):
+ """Get render executable name.
+
+ Get paths from plugin configuration, find executable and return it.
+
+ Returns:
+ (str): Render executable.
+
+ """
+ oiiotool_exe_list = self.GetConfigEntry("OIIOTool_RenderExecutable")
+ oiiotool_exe = FileUtils.SearchFileList(oiiotool_exe_list)
+
+ if oiiotool_exe == "":
+ self.FailRender(("No file found in the semicolon separated "
+ "list \"{}\". The path to the render executable "
+ "can be configured from the Plugin Configuration "
+ "in the Deadline Monitor.").format(
+ oiiotool_exe_list))
+
+ return oiiotool_exe
+
+ def render_argument(self):
+ """Generate command line arguments for render executable.
+
+ Returns:
+ (str): arguments to add to render executable.
+
+ """
+ # Read tile config file. This file is in compatible format with
+ # Draft Tile Assembler
+ data = {}
+ with open(self.config_file, "rU") as f:
+ for text in f:
+ # Parsing key-value pair and removing white-space
+ # around the entries
+ info = [x.strip() for x in text.split("=", 1)]
+
+ if len(info) > 1:
+ try:
+ data[str(info[0])] = info[1]
+ except Exception as e:
+ # should never be called
+ self.FailRender(
+ "Cannot parse config file: {}".format(e))
+
+ # Get output file. We support only EXRs now.
+ output_file = data["ImageFileName"]
+ output_file = RepositoryUtils.CheckPathMapping(output_file)
+ output_file = self.process_path(output_file)
+ """
+ _, ext = os.path.splitext(output_file)
+ if "exr" not in ext:
+ self.FailRender(
+ "[{}] Only EXR format is supported for now.".format(ext))
+ """
+ tile_info = []
+ for tile in range(int(data["TileCount"])):
+ tile_info.append({
+ "filepath": data["Tile{}".format(tile)],
+ "pos_x": int(data["Tile{}X".format(tile)]),
+ "pos_y": int(data["Tile{}Y".format(tile)]),
+ "height": int(data["Tile{}Height".format(tile)]),
+ "width": int(data["Tile{}Width".format(tile)])
+ })
+
+ # FFMpeg doesn't support tile coordinates at the moment.
+ # arguments = self.tile_completer_ffmpeg_args(
+ # int(data["ImageWidth"]), int(data["ImageHeight"]),
+ # tile_info, output_file)
+
+ arguments = self.tile_oiio_args(
+ int(data["ImageWidth"]), int(data["ImageHeight"]),
+ tile_info, output_file)
+ self.LogInfo(
+ "Using arguments: {}".format(" ".join(arguments)))
+ self.tiles = tile_info
+ return " ".join(arguments)
+
+ def process_path(self, filepath):
+ """Handle slashes in file paths."""
+ if SystemUtils.IsRunningOnWindows():
+ filepath = filepath.replace("/", "\\")
+ if filepath.startswith("\\") and not filepath.startswith("\\\\"):
+ filepath = "\\" + filepath
+ else:
+ filepath = filepath.replace("\\", "/")
+ return filepath
+
+ def pre_render_tasks(self):
+ """Load config file and do remapping."""
+ self.LogInfo("OpenPype Tile Assembler starting...")
+ scene_filename = self.GetDataFilename()
+
+ temp_scene_directory = self.CreateTempDirectory(
+ "thread" + str(self.GetThreadNumber()))
+ temp_scene_filename = Path.GetFileName(scene_filename)
+ self.config_file = Path.Combine(
+ temp_scene_directory, temp_scene_filename)
+
+ if SystemUtils.IsRunningOnWindows():
+ RepositoryUtils.CheckPathMappingInFileAndReplaceSeparator(
+ scene_filename, self.config_file, "/", "\\")
+ else:
+ RepositoryUtils.CheckPathMappingInFileAndReplaceSeparator(
+ scene_filename, self.config_file, "\\", "/")
+ os.chmod(self.config_file, os.stat(self.config_file).st_mode)
+
+ def post_render_tasks(self):
+ """Cleanup tiles if required."""
+ if self.GetBooleanPluginInfoEntryWithDefault("CleanupTiles", False):
+ self.LogInfo("Cleaning up Tiles...")
+ for tile in self.tiles:
+ try:
+ self.LogInfo("Deleting: {}".format(tile["filepath"]))
+ os.remove(tile["filepath"])
+ # By this time we would have errored out
+ # if error on missing was enabled
+ except KeyError:
+ pass
+ except OSError:
+ self.LogInfo("Failed to delete: {}".format(
+ tile["filepath"]))
+ pass
+
+ self.LogInfo("OpenPype Tile Assembler Job finished.")
+
+ def handle_stdout_error(self):
+ """Handle errors in stdout."""
+ self.FailRender(self.GetRegexMatch(0))
+
+ def tile_oiio_args(
+ self, output_width, output_height, tile_info, output_path):
+ """Generate oiio tool arguments for tile assembly.
+
+ Args:
+ output_width (int): Width of output image.
+ output_height (int): Height of output image.
+ tiles_info (list): List of tile items, each item must be
+ dictionary with `filepath`, `pos_x` and `pos_y` keys
+ representing path to file and x, y coordinates on output
+ image where top-left point of tile item should start.
+ output_path (str): Path to file where should be output stored.
+
+ Returns:
+ (list): oiio tools arguments.
+
+ """
+ args = []
+
+ # Create new image with output resolution, and with same type and
+ # channels as input
+ oiiotool_path = self.render_executable()
+ first_tile_path = tile_info[0]["filepath"]
+ first_tile_info = info_about_input(oiiotool_path, first_tile_path)
+ create_arg_template = "--create{} {}x{} {}"
+
+ image_type = ""
+ image_format = first_tile_info.get("format")
+ if image_format:
+ image_type = ":type={}".format(image_format)
+
+ create_arg = create_arg_template.format(
+ image_type, output_width,
+ output_height, first_tile_info["nchannels"]
+ )
+ args.append(create_arg)
+
+ for tile in tile_info:
+ path = tile["filepath"]
+ pos_x = tile["pos_x"]
+ tile_height = info_about_input(oiiotool_path, path)["height"]
+ if self.renderer == "vray":
+ pos_y = tile["pos_y"]
+ else:
+ pos_y = output_height - tile["pos_y"] - tile_height
+
+ # Add input path and make sure inputs origin is 0, 0
+ args.append(path)
+ args.append("--origin +0+0")
+ # Swap to have input as foreground
+ args.append("--swap")
+ # Paste foreground to background
+ args.append("--paste {x:+d}{y:+d}".format(x=pos_x, y=pos_y))
+
+ args.append("-o")
+ args.append(output_path)
+
+ return args
+
+ def tile_completer_ffmpeg_args(
+ self, output_width, output_height, tiles_info, output_path):
+ """Generate ffmpeg arguments for tile assembly.
+
+ Expected inputs are tiled images.
+
+ Args:
+ output_width (int): Width of output image.
+ output_height (int): Height of output image.
+ tiles_info (list): List of tile items, each item must be
+ dictionary with `filepath`, `pos_x` and `pos_y` keys
+ representing path to file and x, y coordinates on output
+ image where top-left point of tile item should start.
+ output_path (str): Path to file where should be output stored.
+
+ Returns:
+ (list): ffmpeg arguments.
+
+ """
+ previous_name = "base"
+ ffmpeg_args = []
+ filter_complex_strs = []
+
+ filter_complex_strs.append("nullsrc=size={}x{}[{}]".format(
+ output_width, output_height, previous_name
+ ))
+
+ new_tiles_info = {}
+ for idx, tile_info in enumerate(tiles_info):
+ # Add input and store input index
+ filepath = tile_info["filepath"]
+ ffmpeg_args.append("-i \"{}\"".format(filepath.replace("\\", "/")))
+
+ # Prepare initial filter complex arguments
+ index_name = "input{}".format(idx)
+ filter_complex_strs.append(
+ "[{}]setpts=PTS-STARTPTS[{}]".format(idx, index_name)
+ )
+ tile_info["index"] = idx
+ new_tiles_info[index_name] = tile_info
+
+ # Set frames to 1
+ ffmpeg_args.append("-frames 1")
+
+ # Concatenation filter complex arguments
+ global_index = 1
+ total_index = len(new_tiles_info)
+ for index_name, tile_info in new_tiles_info.items():
+ item_str = (
+ "[{previous_name}][{index_name}]overlay={pos_x}:{pos_y}"
+ ).format(
+ previous_name=previous_name,
+ index_name=index_name,
+ pos_x=tile_info["pos_x"],
+ pos_y=tile_info["pos_y"]
+ )
+ new_previous = "tmp{}".format(global_index)
+ if global_index != total_index:
+ item_str += "[{}]".format(new_previous)
+ filter_complex_strs.append(item_str)
+ previous_name = new_previous
+ global_index += 1
+
+ joined_parts = ";".join(filter_complex_strs)
+ filter_complex_str = "-filter_complex \"{}\"".format(joined_parts)
+
+ ffmpeg_args.append(filter_complex_str)
+ ffmpeg_args.append("-y")
+ ffmpeg_args.append("\"{}\"".format(output_path))
+
+ return ffmpeg_args
diff --git a/openpype/modules/deadline/repository/readme.md b/openpype/modules/deadline/repository/readme.md
new file mode 100644
index 00000000000..31ffffd0b78
--- /dev/null
+++ b/openpype/modules/deadline/repository/readme.md
@@ -0,0 +1,29 @@
+## OpenPype Deadline repository overlay
+
+ This directory is an overlay for Deadline repository.
+ It means that you can copy the whole hierarchy to Deadline repository and it
+ should work.
+
+ Logic:
+ -----
+ GlobalJobPreLoad
+ -----
+
+The `GlobalJobPreLoad` will retrieve the OpenPype executable path from the
+`OpenPype` Deadline Plug-in's settings. Then it will call the executable to
+retrieve the environment variables needed for the Deadline Job.
+These environment variables are injected into rendering process.
+
+Deadline triggers the `GlobalJobPreLoad.py` for each Worker as it starts the
+Job.
+
+*Note*: It also contains backward compatible logic to preserve functionality
+for old Pype2 and non-OpenPype triggered jobs.
+
+ Plugin
+ ------
+ For each render and publishing job the `OpenPype` Deadline Plug-in is checked
+ for the configured location of the OpenPype executable (needs to be configured
+ in `Deadline's Configure Plugins > OpenPype`) through `GlobalJobPreLoad`.
+
+
diff --git a/openpype/modules/default_modules/deadline/deadline_module.py b/openpype/modules/default_modules/deadline/deadline_module.py
deleted file mode 100644
index 1a179e9aafe..00000000000
--- a/openpype/modules/default_modules/deadline/deadline_module.py
+++ /dev/null
@@ -1,34 +0,0 @@
-import os
-from openpype.modules import OpenPypeModule
-from openpype_interfaces import IPluginPaths
-
-
-class DeadlineModule(OpenPypeModule, IPluginPaths):
- name = "deadline"
-
- def __init__(self, manager, settings):
- self.deadline_urls = {}
- super(DeadlineModule, self).__init__(manager, settings)
-
- def initialize(self, modules_settings):
- # This module is always enabled
- deadline_settings = modules_settings[self.name]
- self.enabled = deadline_settings["enabled"]
- deadline_url = deadline_settings.get("DEADLINE_REST_URL")
- if deadline_url:
- self.deadline_urls = {"default": deadline_url}
- else:
- self.deadline_urls = deadline_settings.get("deadline_urls") # noqa: E501
-
- if not self.deadline_urls:
- self.enabled = False
- self.log.warning(("default Deadline Webservice URL "
- "not specified. Disabling module."))
- return
-
- def get_plugin_paths(self):
- """Deadline plugin paths."""
- current_dir = os.path.dirname(os.path.abspath(__file__))
- return {
- "publish": [os.path.join(current_dir, "plugins", "publish")]
- }
diff --git a/openpype/modules/default_modules/deadline/plugins/publish/submit_aftereffects_deadline.py b/openpype/modules/default_modules/deadline/plugins/publish/submit_aftereffects_deadline.py
deleted file mode 100644
index 69159fda1ab..00000000000
--- a/openpype/modules/default_modules/deadline/plugins/publish/submit_aftereffects_deadline.py
+++ /dev/null
@@ -1,121 +0,0 @@
-from openpype.lib import abstract_submit_deadline
-from openpype.lib.abstract_submit_deadline import DeadlineJobInfo
-import pyblish.api
-import os
-import attr
-import getpass
-from avalon import api
-
-
-@attr.s
-class DeadlinePluginInfo():
- Comp = attr.ib(default=None)
- SceneFile = attr.ib(default=None)
- OutputFilePath = attr.ib(default=None)
- Output = attr.ib(default=None)
- StartupDirectory = attr.ib(default=None)
- Arguments = attr.ib(default=None)
- ProjectPath = attr.ib(default=None)
- AWSAssetFile0 = attr.ib(default=None)
- Version = attr.ib(default=None)
- MultiProcess = attr.ib(default=None)
-
-
-class AfterEffectsSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline):
-
- label = "Submit AE to Deadline"
- order = pyblish.api.IntegratorOrder + 0.1
- hosts = ["aftereffects"]
- families = ["render.farm"] # cannot be "render' as that is integrated
- use_published = True
-
- chunk_size = 1000000
-
- def get_job_info(self):
- dln_job_info = DeadlineJobInfo(Plugin="AfterEffects")
-
- context = self._instance.context
-
- dln_job_info.Name = self._instance.data["name"]
- dln_job_info.BatchName = os.path.basename(self._instance.
- data["source"])
- dln_job_info.Plugin = "AfterEffects"
- dln_job_info.UserName = context.data.get(
- "deadlineUser", getpass.getuser())
- if self._instance.data["frameEnd"] > self._instance.data["frameStart"]:
- # Deadline requires integers in frame range
- frame_range = "{}-{}".format(
- int(round(self._instance.data["frameStart"])),
- int(round(self._instance.data["frameEnd"])))
- dln_job_info.Frames = frame_range
-
- dln_job_info.ChunkSize = self.chunk_size
- dln_job_info.OutputFilename = \
- os.path.basename(self._instance.data["expectedFiles"][0])
- dln_job_info.OutputDirectory = \
- os.path.dirname(self._instance.data["expectedFiles"][0])
- dln_job_info.JobDelay = "00:00:00"
-
- keys = [
- "FTRACK_API_KEY",
- "FTRACK_API_USER",
- "FTRACK_SERVER",
- "AVALON_PROJECT",
- "AVALON_ASSET",
- "AVALON_TASK",
- "AVALON_APP_NAME",
- "OPENPYPE_DEV",
- "OPENPYPE_LOG_NO_COLORS"
- ]
-
- environment = dict({key: os.environ[key] for key in keys
- if key in os.environ}, **api.Session)
- for key in keys:
- val = environment.get(key)
- if val:
- dln_job_info.EnvironmentKeyValue = "{key}={value}".format(
- key=key,
- value=val)
- # to recognize job from PYPE for turning Event On/Off
- dln_job_info.EnvironmentKeyValue = "OPENPYPE_RENDER_JOB=1"
-
- return dln_job_info
-
- def get_plugin_info(self):
- deadline_plugin_info = DeadlinePluginInfo()
- context = self._instance.context
- script_path = context.data["currentFile"]
-
- render_path = self._instance.data["expectedFiles"][0]
-
- if len(self._instance.data["expectedFiles"]) > 1:
- # replace frame ('000001') with Deadline's required '[#######]'
- # expects filename in format project_asset_subset_version.FRAME.ext
- render_dir = os.path.dirname(render_path)
- file_name = os.path.basename(render_path)
- arr = file_name.split('.')
- assert len(arr) == 3, \
- "Unable to parse frames from {}".format(file_name)
- hashed = '[{}]'.format(len(arr[1]) * "#")
-
- render_path = os.path.join(render_dir,
- '{}.{}.{}'.format(arr[0], hashed,
- arr[2]))
-
- deadline_plugin_info.MultiProcess = True
- deadline_plugin_info.Comp = self._instance.data["comp_name"]
- deadline_plugin_info.Version = "17.5"
- deadline_plugin_info.SceneFile = self.scene_path
- deadline_plugin_info.Output = render_path.replace("\\", "/")
-
- return attr.asdict(deadline_plugin_info)
-
- def from_published_scene(self):
- """ Do not overwrite expected files.
-
- Use published is set to True, so rendering will be triggered
- from published scene (in 'publish' folder). Default implementation
- of abstract class renames expected (eg. rendered) files accordingly
- which is not needed here.
- """
- return super().from_published_scene(False)
diff --git a/openpype/modules/default_modules/deadline/plugins/publish/submit_maya_deadline.py b/openpype/modules/default_modules/deadline/plugins/publish/submit_maya_deadline.py
deleted file mode 100644
index 51a19e2aadf..00000000000
--- a/openpype/modules/default_modules/deadline/plugins/publish/submit_maya_deadline.py
+++ /dev/null
@@ -1,1147 +0,0 @@
-# -*- coding: utf-8 -*-
-"""Submitting render job to Deadline.
-
-This module is taking care of submitting job from Maya to Deadline. It
-creates job and set correct environments. Its behavior is controlled by
-``DEADLINE_REST_URL`` environment variable - pointing to Deadline Web Service
-and :data:`MayaSubmitDeadline.use_published` property telling Deadline to
-use published scene workfile or not.
-
-If ``vrscene`` or ``assscene`` are detected in families, it will first
-submit job to export these files and then dependent job to render them.
-
-Attributes:
- payload_skeleton (dict): Skeleton payload data sent as job to Deadline.
- Default values are for ``MayaBatch`` plugin.
-
-"""
-
-from __future__ import print_function
-import os
-import json
-import getpass
-import copy
-import re
-import hashlib
-from datetime import datetime
-import itertools
-from collections import OrderedDict
-
-import clique
-import requests
-
-from maya import cmds
-
-from avalon import api
-import pyblish.api
-
-from openpype.hosts.maya.api import lib
-
-# Documentation for keys available at:
-# https://docs.thinkboxsoftware.com
-# /products/deadline/8.0/1_User%20Manual/manual
-# /manual-submission.html#job-info-file-options
-
-payload_skeleton_template = {
- "JobInfo": {
- "BatchName": None, # Top-level group name
- "Name": None, # Job name, as seen in Monitor
- "UserName": None,
- "Plugin": "MayaBatch",
- "Frames": "{start}-{end}x{step}",
- "Comment": None,
- "Priority": 50,
- },
- "PluginInfo": {
- "SceneFile": None, # Input
- "OutputFilePath": None, # Output directory and filename
- "OutputFilePrefix": None,
- "Version": cmds.about(version=True), # Mandatory for Deadline
- "UsingRenderLayers": True,
- "RenderLayer": None, # Render only this layer
- "Renderer": None,
- "ProjectPath": None, # Resolve relative references
- },
- "AuxFiles": [] # Mandatory for Deadline, may be empty
-}
-
-
-def _format_tiles(
- filename, index, tiles_x, tiles_y,
- width, height, prefix):
- """Generate tile entries for Deadline tile job.
-
- Returns two dictionaries - one that can be directly used in Deadline
- job, second that can be used for Deadline Assembly job configuration
- file.
-
- This will format tile names:
-
- Example::
- {
- "OutputFilename0Tile0": "_tile_1x1_4x4_Main_beauty.1001.exr",
- "OutputFilename0Tile1": "_tile_2x1_4x4_Main_beauty.1001.exr"
- }
-
- And add tile prefixes like:
-
- Example::
- Image prefix is:
- `maya///_`
-
- Result for tile 0 for 4x4 will be:
- `maya///_tile_1x1_4x4__`
-
- Calculating coordinates is tricky as in Job they are defined as top,
- left, bottom, right with zero being in top-left corner. But Assembler
- configuration file takes tile coordinates as X, Y, Width and Height and
- zero is bottom left corner.
-
- Args:
- filename (str): Filename to process as tiles.
- index (int): Index of that file if it is sequence.
- tiles_x (int): Number of tiles in X.
- tiles_y (int): Number if tikes in Y.
- width (int): Width resolution of final image.
- height (int): Height resolution of final image.
- prefix (str): Image prefix.
-
- Returns:
- (dict, dict): Tuple of two dictionaires - first can be used to
- extend JobInfo, second has tiles x, y, width and height
- used for assembler configuration.
-
- """
- tile = 0
- out = {"JobInfo": {}, "PluginInfo": {}}
- cfg = OrderedDict()
- w_space = width / tiles_x
- h_space = height / tiles_y
-
- cfg["TilesCropped"] = "False"
-
- for tile_x in range(1, tiles_x + 1):
- for tile_y in reversed(range(1, tiles_y + 1)):
- tile_prefix = "_tile_{}x{}_{}x{}_".format(
- tile_x, tile_y,
- tiles_x,
- tiles_y
- )
- out_tile_index = "OutputFilename{}Tile{}".format(
- str(index), tile
- )
- new_filename = "{}/{}{}".format(
- os.path.dirname(filename),
- tile_prefix,
- os.path.basename(filename)
- )
- out["JobInfo"][out_tile_index] = new_filename
- out["PluginInfo"]["RegionPrefix{}".format(str(tile))] = \
- "/{}".format(tile_prefix).join(prefix.rsplit("/", 1))
-
- out["PluginInfo"]["RegionTop{}".format(tile)] = int(height) - (tile_y * h_space) # noqa: E501
- out["PluginInfo"]["RegionBottom{}".format(tile)] = int(height) - ((tile_y - 1) * h_space) - 1 # noqa: E501
- out["PluginInfo"]["RegionLeft{}".format(tile)] = (tile_x - 1) * w_space # noqa: E501
- out["PluginInfo"]["RegionRight{}".format(tile)] = (tile_x * w_space) - 1 # noqa: E501
-
- cfg["Tile{}".format(tile)] = new_filename
- cfg["Tile{}Tile".format(tile)] = new_filename
- cfg["Tile{}FileName".format(tile)] = new_filename
- cfg["Tile{}X".format(tile)] = (tile_x - 1) * w_space
-
- cfg["Tile{}Y".format(tile)] = int(height) - (tile_y * h_space)
-
- cfg["Tile{}Width".format(tile)] = w_space
- cfg["Tile{}Height".format(tile)] = h_space
-
- tile += 1
- return out, cfg
-
-
-def get_renderer_variables(renderlayer, root):
- """Retrieve the extension which has been set in the VRay settings.
-
- Will return None if the current renderer is not VRay
- For Maya 2016.5 and up the renderSetup creates renderSetupLayer node which
- start with `rs`. Use the actual node name, do NOT use the `nice name`
-
- Args:
- renderlayer (str): the node name of the renderlayer.
- root (str): base path to render
-
- Returns:
- dict
-
- """
- renderer = lib.get_renderer(renderlayer or lib.get_current_renderlayer())
- render_attrs = lib.RENDER_ATTRS.get(renderer, lib.RENDER_ATTRS["default"])
-
- padding = cmds.getAttr("{}.{}".format(render_attrs["node"],
- render_attrs["padding"]))
-
- filename_0 = cmds.renderSettings(
- fullPath=True,
- gin="#" * int(padding),
- lut=True,
- layer=renderlayer or lib.get_current_renderlayer())[0]
- filename_0 = re.sub('_', '_beauty',
- filename_0, flags=re.IGNORECASE)
- prefix_attr = "defaultRenderGlobals.imageFilePrefix"
- if renderer == "vray":
- renderlayer = renderlayer.split("_")[-1]
- # Maya's renderSettings function does not return V-Ray file extension
- # so we get the extension from vraySettings
- extension = cmds.getAttr("vraySettings.imageFormatStr")
-
- # When V-Ray image format has not been switched once from default .png
- # the getAttr command above returns None. As such we explicitly set
- # it to `.png`
- if extension is None:
- extension = "png"
-
- if extension in ["exr (multichannel)", "exr (deep)"]:
- extension = "exr"
-
- prefix_attr = "vraySettings.fileNamePrefix"
- filename_prefix = cmds.getAttr(prefix_attr)
- # we need to determine path for vray as maya `renderSettings` query
- # does not work for vray.
- scene = cmds.file(query=True, sceneName=True)
- scene, _ = os.path.splitext(os.path.basename(scene))
- filename_0 = re.sub('', scene, filename_prefix, flags=re.IGNORECASE) # noqa: E501
- filename_0 = re.sub('', renderlayer, filename_0, flags=re.IGNORECASE) # noqa: E501
- filename_0 = "{}.{}.{}".format(
- filename_0, "#" * int(padding), extension)
- filename_0 = os.path.normpath(os.path.join(root, filename_0))
- elif renderer == "renderman":
- prefix_attr = "rmanGlobals.imageFileFormat"
- elif renderer == "redshift":
- # mapping redshift extension dropdown values to strings
- ext_mapping = ["iff", "exr", "tif", "png", "tga", "jpg"]
- extension = ext_mapping[
- cmds.getAttr("redshiftOptions.imageFormat")
- ]
- else:
- # Get the extension, getAttr defaultRenderGlobals.imageFormat
- # returns an index number.
- filename_base = os.path.basename(filename_0)
- extension = os.path.splitext(filename_base)[-1].strip(".")
-
- filename_prefix = cmds.getAttr(prefix_attr)
- return {"ext": extension,
- "filename_prefix": filename_prefix,
- "padding": padding,
- "filename_0": filename_0}
-
-
-class MayaSubmitDeadline(pyblish.api.InstancePlugin):
- """Submit available render layers to Deadline.
-
- Renders are submitted to a Deadline Web Service as
- supplied via settings key "DEADLINE_REST_URL".
-
- Attributes:
- use_published (bool): Use published scene to render instead of the
- one in work area.
-
- """
-
- label = "Submit to Deadline"
- order = pyblish.api.IntegratorOrder + 0.1
- hosts = ["maya"]
- families = ["renderlayer"]
-
- use_published = True
- tile_assembler_plugin = "OpenPypeTileAssembler"
- asset_dependencies = False
- limit_groups = []
- group = "none"
-
- def process(self, instance):
- """Plugin entry point."""
- instance.data["toBeRenderedOn"] = "deadline"
- context = instance.context
-
- self._instance = instance
- self.payload_skeleton = copy.deepcopy(payload_skeleton_template)
-
- # get default deadline webservice url from deadline module
- self.deadline_url = instance.context.data.get("defaultDeadline")
- # if custom one is set in instance, use that
- if instance.data.get("deadlineUrl"):
- self.deadline_url = instance.data.get("deadlineUrl")
- assert self.deadline_url, "Requires Deadline Webservice URL"
-
- self._job_info = (
- context.data["project_settings"].get(
- "deadline", {}).get(
- "publish", {}).get(
- "MayaSubmitDeadline", {}).get(
- "jobInfo", {})
- )
-
- self._plugin_info = (
- context.data["project_settings"].get(
- "deadline", {}).get(
- "publish", {}).get(
- "MayaSubmitDeadline", {}).get(
- "pluginInfo", {})
- )
-
- self.limit_groups = (
- context.data["project_settings"].get(
- "deadline", {}).get(
- "publish", {}).get(
- "MayaSubmitDeadline", {}).get(
- "limit", [])
- )
-
- self.group = (
- context.data["project_settings"].get(
- "deadline", {}).get(
- "publish", {}).get(
- "MayaSubmitDeadline", {}).get(
- "group", "none")
- )
-
- context = instance.context
- workspace = context.data["workspaceDir"]
- anatomy = context.data['anatomy']
- instance.data["toBeRenderedOn"] = "deadline"
-
- filepath = None
- patches = (
- context.data["project_settings"].get(
- "deadline", {}).get(
- "publish", {}).get(
- "MayaSubmitDeadline", {}).get(
- "scene_patches", {})
- )
-
- # Handle render/export from published scene or not ------------------
- if self.use_published:
- patched_files = []
- for i in context:
- if "workfile" not in i.data["families"]:
- continue
- assert i.data["publish"] is True, (
- "Workfile (scene) must be published along")
- template_data = i.data.get("anatomyData")
- rep = i.data.get("representations")[0].get("name")
- template_data["representation"] = rep
- template_data["ext"] = rep
- template_data["comment"] = None
- anatomy_filled = anatomy.format(template_data)
- template_filled = anatomy_filled["publish"]["path"]
- filepath = os.path.normpath(template_filled)
- self.log.info("Using published scene for render {}".format(
- filepath))
-
- if not os.path.exists(filepath):
- self.log.error("published scene does not exist!")
- raise
- # now we need to switch scene in expected files
- # because token will now point to published
- # scene file and that might differ from current one
- new_scene = os.path.splitext(
- os.path.basename(filepath))[0]
- orig_scene = os.path.splitext(
- os.path.basename(context.data["currentFile"]))[0]
- exp = instance.data.get("expectedFiles")
-
- if isinstance(exp[0], dict):
- # we have aovs and we need to iterate over them
- new_exp = {}
- for aov, files in exp[0].items():
- replaced_files = []
- for f in files:
- replaced_files.append(
- f.replace(orig_scene, new_scene)
- )
- new_exp[aov] = replaced_files
- instance.data["expectedFiles"] = [new_exp]
- else:
- new_exp = []
- for f in exp:
- new_exp.append(
- f.replace(orig_scene, new_scene)
- )
- instance.data["expectedFiles"] = [new_exp]
-
- if instance.data.get("publishRenderMetadataFolder"):
- instance.data["publishRenderMetadataFolder"] = \
- instance.data["publishRenderMetadataFolder"].replace(
- orig_scene, new_scene)
- self.log.info("Scene name was switched {} -> {}".format(
- orig_scene, new_scene
- ))
- # patch workfile is needed
- if filepath not in patched_files:
- patched_file = self._patch_workfile(filepath, patches)
- patched_files.append(patched_file)
-
- all_instances = []
- for result in context.data["results"]:
- if (result["instance"] is not None and
- result["instance"] not in all_instances): # noqa: E128
- all_instances.append(result["instance"])
-
- # fallback if nothing was set
- if not filepath:
- self.log.warning("Falling back to workfile")
- filepath = context.data["currentFile"]
-
- self.log.debug(filepath)
-
- # Gather needed data ------------------------------------------------
- default_render_file = instance.context.data.get('project_settings')\
- .get('maya')\
- .get('create')\
- .get('CreateRender')\
- .get('default_render_image_folder')
- filename = os.path.basename(filepath)
- comment = context.data.get("comment", "")
- dirname = os.path.join(workspace, default_render_file)
- renderlayer = instance.data['setMembers'] # rs_beauty
- deadline_user = context.data.get("user", getpass.getuser())
- jobname = "%s - %s" % (filename, instance.name)
-
- # Get the variables depending on the renderer
- render_variables = get_renderer_variables(renderlayer, dirname)
- filename_0 = render_variables["filename_0"]
- if self.use_published:
- new_scene = os.path.splitext(filename)[0]
- orig_scene = os.path.splitext(
- os.path.basename(context.data["currentFile"]))[0]
- filename_0 = render_variables["filename_0"].replace(
- orig_scene, new_scene)
-
- output_filename_0 = filename_0
-
- # Create render folder ----------------------------------------------
- try:
- # Ensure render folder exists
- os.makedirs(dirname)
- except OSError:
- pass
-
- # Fill in common data to payload ------------------------------------
- payload_data = {}
- payload_data["filename"] = filename
- payload_data["filepath"] = filepath
- payload_data["jobname"] = jobname
- payload_data["deadline_user"] = deadline_user
- payload_data["comment"] = comment
- payload_data["output_filename_0"] = output_filename_0
- payload_data["render_variables"] = render_variables
- payload_data["renderlayer"] = renderlayer
- payload_data["workspace"] = workspace
- payload_data["dirname"] = dirname
-
- self.log.info("--- Submission data:")
- for k, v in payload_data.items():
- self.log.info("- {}: {}".format(k, v))
- self.log.info("-" * 20)
-
- frame_pattern = self.payload_skeleton["JobInfo"]["Frames"]
- self.payload_skeleton["JobInfo"]["Frames"] = frame_pattern.format(
- start=int(self._instance.data["frameStartHandle"]),
- end=int(self._instance.data["frameEndHandle"]),
- step=int(self._instance.data["byFrameStep"]))
-
- self.payload_skeleton["JobInfo"]["Plugin"] = self._instance.data.get(
- "mayaRenderPlugin", "MayaBatch")
-
- self.payload_skeleton["JobInfo"]["BatchName"] = filename
- # Job name, as seen in Monitor
- self.payload_skeleton["JobInfo"]["Name"] = jobname
- # Arbitrary username, for visualisation in Monitor
- self.payload_skeleton["JobInfo"]["UserName"] = deadline_user
- # Set job priority
- self.payload_skeleton["JobInfo"]["Priority"] = \
- self._instance.data.get("priority", 50)
-
- if self.group != "none" and self.group:
- self.payload_skeleton["JobInfo"]["Group"] = self.group
-
- if self.limit_groups:
- self.payload_skeleton["JobInfo"]["LimitGroups"] = \
- ",".join(self.limit_groups)
- # Optional, enable double-click to preview rendered
- # frames from Deadline Monitor
- self.payload_skeleton["JobInfo"]["OutputDirectory0"] = \
- os.path.dirname(output_filename_0).replace("\\", "/")
- self.payload_skeleton["JobInfo"]["OutputFilename0"] = \
- output_filename_0.replace("\\", "/")
-
- self.payload_skeleton["JobInfo"]["Comment"] = comment
- self.payload_skeleton["PluginInfo"]["RenderLayer"] = renderlayer
-
- # Adding file dependencies.
- dependencies = instance.context.data["fileDependencies"]
- dependencies.append(filepath)
- if self.asset_dependencies:
- for dependency in dependencies:
- key = "AssetDependency" + str(dependencies.index(dependency))
- self.payload_skeleton["JobInfo"][key] = dependency
-
- # Handle environments -----------------------------------------------
- # We need those to pass them to pype for it to set correct context
- keys = [
- "FTRACK_API_KEY",
- "FTRACK_API_USER",
- "FTRACK_SERVER",
- "AVALON_PROJECT",
- "AVALON_ASSET",
- "AVALON_TASK",
- "AVALON_APP_NAME",
- "OPENPYPE_DEV",
- "OPENPYPE_LOG_NO_COLORS"
- ]
-
- environment = dict({key: os.environ[key] for key in keys
- if key in os.environ}, **api.Session)
- environment["OPENPYPE_LOG_NO_COLORS"] = "1"
- environment["OPENPYPE_MAYA_VERSION"] = cmds.about(v=True)
- # to recognize job from PYPE for turning Event On/Off
- environment["OPENPYPE_RENDER_JOB"] = "1"
- self.payload_skeleton["JobInfo"].update({
- "EnvironmentKeyValue%d" % index: "{key}={value}".format(
- key=key,
- value=environment[key]
- ) for index, key in enumerate(environment)
- })
- # Add options from RenderGlobals-------------------------------------
- render_globals = instance.data.get("renderGlobals", {})
- self.payload_skeleton["JobInfo"].update(render_globals)
-
- # Submit preceding export jobs -------------------------------------
- export_job = None
- assert not all(x in instance.data["families"]
- for x in ['vrayscene', 'assscene']), (
- "Vray Scene and Ass Scene options are mutually exclusive")
- if "vrayscene" in instance.data["families"]:
- export_job = self._submit_export(payload_data, "vray")
-
- if "assscene" in instance.data["families"]:
- export_job = self._submit_export(payload_data, "arnold")
-
- # Prepare main render job -------------------------------------------
- if "vrayscene" in instance.data["families"]:
- payload = self._get_vray_render_payload(payload_data)
- elif "assscene" in instance.data["families"]:
- payload = self._get_arnold_render_payload(payload_data)
- else:
- payload = self._get_maya_payload(payload_data)
-
- # Add export job as dependency --------------------------------------
- if export_job:
- payload["JobInfo"]["JobDependency0"] = export_job
-
- # Add list of expected files to job ---------------------------------
- exp = instance.data.get("expectedFiles")
- exp_index = 0
- output_filenames = {}
-
- if isinstance(exp[0], dict):
- # we have aovs and we need to iterate over them
- for _aov, files in exp[0].items():
- col, rem = clique.assemble(files)
- if not col and rem:
- # we couldn't find any collections but have
- # individual files.
- assert len(rem) == 1, ("Found multiple non related files "
- "to render, don't know what to do "
- "with them.")
- output_file = rem[0]
- if not instance.data.get("tileRendering"):
- payload['JobInfo']['OutputFilename' + str(exp_index)] = output_file # noqa: E501
- else:
- output_file = col[0].format('{head}{padding}{tail}')
- if not instance.data.get("tileRendering"):
- payload['JobInfo']['OutputFilename' + str(exp_index)] = output_file # noqa: E501
-
- output_filenames['OutputFilename' + str(exp_index)] = output_file # noqa: E501
- exp_index += 1
- else:
- col, rem = clique.assemble(exp)
- if not col and rem:
- # we couldn't find any collections but have
- # individual files.
- assert len(rem) == 1, ("Found multiple non related files "
- "to render, don't know what to do "
- "with them.")
-
- output_file = rem[0]
- if not instance.data.get("tileRendering"):
- payload['JobInfo']['OutputFilename' + str(exp_index)] = output_file # noqa: E501
- else:
- output_file = col[0].format('{head}{padding}{tail}')
- if not instance.data.get("tileRendering"):
- payload['JobInfo']['OutputFilename' + str(exp_index)] = output_file # noqa: E501
-
- output_filenames['OutputFilename' + str(exp_index)] = output_file
-
- plugin = payload["JobInfo"]["Plugin"]
- self.log.info("using render plugin : {}".format(plugin))
-
- # Store output dir for unified publisher (filesequence)
- instance.data["outputDir"] = os.path.dirname(output_filename_0)
-
- self.preflight_check(instance)
-
- # add jobInfo and pluginInfo variables from Settings
- payload["JobInfo"].update(self._job_info)
- payload["PluginInfo"].update(self._plugin_info)
-
- # Prepare tiles data ------------------------------------------------
- if instance.data.get("tileRendering"):
- # if we have sequence of files, we need to create tile job for
- # every frame
-
- payload["JobInfo"]["TileJob"] = True
- payload["JobInfo"]["TileJobTilesInX"] = instance.data.get("tilesX")
- payload["JobInfo"]["TileJobTilesInY"] = instance.data.get("tilesY")
- payload["PluginInfo"]["ImageHeight"] = instance.data.get("resolutionHeight") # noqa: E501
- payload["PluginInfo"]["ImageWidth"] = instance.data.get("resolutionWidth") # noqa: E501
- payload["PluginInfo"]["RegionRendering"] = True
-
- assembly_payload = {
- "AuxFiles": [],
- "JobInfo": {
- "BatchName": payload["JobInfo"]["BatchName"],
- "Frames": 1,
- "Name": "{} - Tile Assembly Job".format(
- payload["JobInfo"]["Name"]),
- "OutputDirectory0":
- payload["JobInfo"]["OutputDirectory0"].replace(
- "\\", "/"),
- "Plugin": self.tile_assembler_plugin,
- "MachineLimit": 1
- },
- "PluginInfo": {
- "CleanupTiles": 1,
- "ErrorOnMissing": True
- }
- }
- assembly_payload["JobInfo"].update(output_filenames)
- assembly_payload["JobInfo"]["Priority"] = self._instance.data.get(
- "priority", 50)
- assembly_payload["JobInfo"]["UserName"] = deadline_user
-
- frame_payloads = []
- assembly_payloads = []
-
- R_FRAME_NUMBER = re.compile(r".+\.(?P/_/"
-
- scene, _ = os.path.splitext(data["filename"])
- first_file = self.format_vray_output_filename(scene, template)
- first_file = "{}/{}".format(data["workspace"], first_file)
- job_info_ext = {
- "Name": "Render {} [{}-{}]".format(
- data["jobname"],
- int(self._instance.data["frameStartHandle"]),
- int(self._instance.data["frameEndHandle"])),
-
- "Plugin": "Vray",
- "OverrideTaskExtraInfoNames": False,
- }
-
- plugin_info = {
- "InputFilename": first_file,
- "SeparateFilesPerFrame": True,
- "VRayEngine": "V-Ray",
-
- "Width": self._instance.data["resolutionWidth"],
- "Height": self._instance.data["resolutionHeight"],
- "OutputFilePath": payload["JobInfo"]["OutputDirectory0"],
- "OutputFileName": payload["JobInfo"]["OutputFilename0"]
- }
-
- payload["JobInfo"].update(job_info_ext)
- payload["PluginInfo"].update(plugin_info)
- return payload
-
- def _get_arnold_render_payload(self, data):
- payload = copy.deepcopy(self.payload_skeleton)
- ass_file, _ = os.path.splitext(data["output_filename_0"])
- first_file = ass_file + ".ass"
- job_info_ext = {
- "Name": "Render {} [{}-{}]".format(
- data["jobname"],
- int(self._instance.data["frameStartHandle"]),
- int(self._instance.data["frameEndHandle"])),
-
- "Plugin": "Arnold",
- "OverrideTaskExtraInfoNames": False,
- }
-
- plugin_info = {
- "ArnoldFile": first_file,
- }
-
- payload["JobInfo"].update(job_info_ext)
- payload["PluginInfo"].update(plugin_info)
- return payload
-
- def _submit_export(self, data, format):
- if format == "vray":
- payload = self._get_vray_export_payload(data)
- self.log.info("Submitting vrscene export job.")
- elif format == "arnold":
- payload = self._get_arnold_export_payload(data)
- self.log.info("Submitting ass export job.")
-
- url = "{}/api/jobs".format(self.deadline_url)
- response = self._requests_post(url, json=payload)
- if not response.ok:
- self.log.error("Submition failed!")
- self.log.error(response.status_code)
- self.log.error(response.content)
- self.log.debug(payload)
- raise RuntimeError(response.text)
-
- dependency = response.json()
- return dependency["_id"]
-
- def preflight_check(self, instance):
- """Ensure the startFrame, endFrame and byFrameStep are integers."""
- for key in ("frameStartHandle", "frameEndHandle", "byFrameStep"):
- value = instance.data[key]
-
- if int(value) == value:
- continue
-
- self.log.warning(
- "%f=%d was rounded off to nearest integer"
- % (value, int(value))
- )
-
- def _requests_post(self, *args, **kwargs):
- """Wrap request post method.
-
- Disabling SSL certificate validation if ``DONT_VERIFY_SSL`` environment
- variable is found. This is useful when Deadline or Muster server are
- running with self-signed certificates and their certificate is not
- added to trusted certificates on client machines.
-
- Warning:
- Disabling SSL certificate validation is defeating one line
- of defense SSL is providing and it is not recommended.
-
- """
- if 'verify' not in kwargs:
- kwargs['verify'] = not os.getenv("OPENPYPE_DONT_VERIFY_SSL", True)
- # add 10sec timeout before bailing out
- kwargs['timeout'] = 10
- return requests.post(*args, **kwargs)
-
- def _requests_get(self, *args, **kwargs):
- """Wrap request get method.
-
- Disabling SSL certificate validation if ``DONT_VERIFY_SSL`` environment
- variable is found. This is useful when Deadline or Muster server are
- running with self-signed certificates and their certificate is not
- added to trusted certificates on client machines.
-
- Warning:
- Disabling SSL certificate validation is defeating one line
- of defense SSL is providing and it is not recommended.
-
- """
- if 'verify' not in kwargs:
- kwargs['verify'] = not os.getenv("OPENPYPE_DONT_VERIFY_SSL", True)
- # add 10sec timeout before bailing out
- kwargs['timeout'] = 10
- return requests.get(*args, **kwargs)
-
- def format_vray_output_filename(self, filename, template, dir=False):
- """Format the expected output file of the Export job.
-
- Example:
- /_/
- "shot010_v006/shot010_v006_CHARS/CHARS"
-
- Args:
- instance:
- filename(str):
- dir(bool):
-
- Returns:
- str
-
- """
- def smart_replace(string, key_values):
- new_string = string
- for key, value in key_values.items():
- new_string = new_string.replace(key, value)
- return new_string
-
- # Ensure filename has no extension
- file_name, _ = os.path.splitext(filename)
-
- layer = self._instance.data['setMembers']
-
- # Reformat without tokens
- output_path = smart_replace(
- template,
- {"": file_name,
- "": layer})
-
- if dir:
- return output_path.replace("\\", "/")
-
- start_frame = int(self._instance.data["frameStartHandle"])
- filename_zero = "{}_{:04d}.vrscene".format(output_path, start_frame)
-
- result = filename_zero.replace("\\", "/")
-
- return result
-
- def _patch_workfile(self, file, patches):
- # type: (str, dict) -> [str, None]
- """Patch Maya scene.
-
- This will take list of patches (lines to add) and apply them to
- *published* Maya scene file (that is used later for rendering).
-
- Patches are dict with following structure::
- {
- "name": "Name of patch",
- "regex": "regex of line before patch",
- "line": "line to insert"
- }
-
- Args:
- file (str): File to patch.
- patches (dict): Dictionary defining patches.
-
- Returns:
- str: Patched file path or None
-
- """
- if os.path.splitext(file)[1].lower() != ".ma" or not patches:
- return None
-
- compiled_regex = [re.compile(p["regex"]) for p in patches]
- with open(file, "r+") as pf:
- scene_data = pf.readlines()
- for ln, line in enumerate(scene_data):
- for i, r in enumerate(compiled_regex):
- if re.match(r, line):
- scene_data.insert(ln + 1, patches[i]["line"])
- pf.seek(0)
- pf.writelines(scene_data)
- pf.truncate()
- self.log.info(
- "Applied {} patch to scene.".format(
- patches[i]["name"]))
- return file
diff --git a/openpype/modules/default_modules/deadline/plugins/publish/validate_expected_and_rendered_files.py b/openpype/modules/default_modules/deadline/plugins/publish/validate_expected_and_rendered_files.py
deleted file mode 100644
index 719c7dfe3ea..00000000000
--- a/openpype/modules/default_modules/deadline/plugins/publish/validate_expected_and_rendered_files.py
+++ /dev/null
@@ -1,189 +0,0 @@
-import os
-import json
-import requests
-
-import pyblish.api
-
-from openpype.lib.abstract_submit_deadline import requests_get
-from openpype.lib.delivery import collect_frames
-
-
-class ValidateExpectedFiles(pyblish.api.InstancePlugin):
- """Compare rendered and expected files"""
-
- label = "Validate rendered files from Deadline"
- order = pyblish.api.ValidatorOrder
- families = ["render"]
- targets = ["deadline"]
-
- # check if actual frame range on render job wasn't different
- # case when artists wants to render only subset of frames
- allow_user_override = True
-
- def process(self, instance):
- self.instance = instance
- frame_list = self._get_frame_list(instance.data["render_job_id"])
-
- for repre in instance.data["representations"]:
- expected_files = self._get_expected_files(repre)
-
- staging_dir = repre["stagingDir"]
- existing_files = self._get_existing_files(staging_dir)
-
- expected_non_existent = expected_files.difference(
- existing_files)
- if len(expected_non_existent) != 0:
- self.log.info("Some expected files missing {}".format(
- expected_non_existent))
-
- if self.allow_user_override:
- file_name_template, frame_placeholder = \
- self._get_file_name_template_and_placeholder(
- expected_files)
-
- if not file_name_template:
- return
-
- real_expected_rendered = self._get_real_render_expected(
- file_name_template,
- frame_placeholder,
- frame_list)
-
- real_expected_non_existent = \
- real_expected_rendered.difference(existing_files)
- if len(real_expected_non_existent) != 0:
- raise RuntimeError("Still missing some files {}".
- format(real_expected_non_existent))
- self.log.info("Update range from actual job range")
- repre["files"] = sorted(list(real_expected_rendered))
- else:
- raise RuntimeError("Some expected files missing {}".format(
- expected_non_existent))
-
- def _get_frame_list(self, original_job_id):
- """
- Returns list of frame ranges from all render job.
-
- Render job might be requeried so job_id in metadata.json is invalid
- GlobalJobPreload injects current ids to RENDER_JOB_IDS.
-
- Args:
- original_job_id (str)
- Returns:
- (list)
- """
- all_frame_lists = []
- render_job_ids = os.environ.get("RENDER_JOB_IDS")
- if render_job_ids:
- render_job_ids = render_job_ids.split(',')
- else: # fallback
- render_job_ids = [original_job_id]
-
- for job_id in render_job_ids:
- job_info = self._get_job_info(job_id)
- frame_list = job_info["Props"]["Frames"]
- if frame_list:
- all_frame_lists.extend(frame_list.split(','))
-
- return all_frame_lists
-
- def _get_real_render_expected(self, file_name_template, frame_placeholder,
- frame_list):
- """
- Calculates list of names of expected rendered files.
-
- Might be different from job expected files if user explicitly and
- manually change frame list on Deadline job.
- """
- real_expected_rendered = set()
- src_padding_exp = "%0{}d".format(len(frame_placeholder))
- for frames in frame_list:
- if '-' not in frames: # single frame
- frames = "{}-{}".format(frames, frames)
-
- start, end = frames.split('-')
- for frame in range(int(start), int(end) + 1):
- ren_name = file_name_template.replace(
- frame_placeholder, src_padding_exp % frame)
- real_expected_rendered.add(ren_name)
-
- return real_expected_rendered
-
- def _get_file_name_template_and_placeholder(self, files):
- """Returns file name with frame replaced with # and this placeholder"""
- sources_and_frames = collect_frames(files)
-
- file_name_template = frame_placeholder = None
- for file_name, frame in sources_and_frames.items():
- frame_placeholder = "#" * len(frame)
- file_name_template = os.path.basename(
- file_name.replace(frame, frame_placeholder))
- break
-
- return file_name_template, frame_placeholder
-
- def _get_job_info(self, job_id):
- """
- Calls DL for actual job info for 'job_id'
-
- Might be different than job info saved in metadata.json if user
- manually changes job pre/during rendering.
- """
- # get default deadline webservice url from deadline module
- deadline_url = self.instance.context.data["defaultDeadline"]
- # if custom one is set in instance, use that
- if self.instance.data.get("deadlineUrl"):
- deadline_url = self.instance.data.get("deadlineUrl")
- assert deadline_url, "Requires Deadline Webservice URL"
-
- url = "{}/api/jobs?JobID={}".format(deadline_url, job_id)
- try:
- response = requests_get(url)
- except requests.exceptions.ConnectionError:
- print("Deadline is not accessible at {}".format(deadline_url))
- # self.log("Deadline is not accessible at {}".format(deadline_url))
- return {}
-
- if not response.ok:
- self.log.error("Submission failed!")
- self.log.error(response.status_code)
- self.log.error(response.content)
- raise RuntimeError(response.text)
-
- json_content = response.json()
- if json_content:
- return json_content.pop()
- return {}
-
- def _parse_metadata_json(self, json_path):
- if not os.path.exists(json_path):
- msg = "Metadata file {} doesn't exist".format(json_path)
- raise RuntimeError(msg)
-
- with open(json_path) as fp:
- try:
- return json.load(fp)
- except Exception as exc:
- self.log.error(
- "Error loading json: "
- "{} - Exception: {}".format(json_path, exc)
- )
-
- def _get_existing_files(self, out_dir):
- """Returns set of existing file names from 'out_dir'"""
- existing_files = set()
- for file_name in os.listdir(out_dir):
- existing_files.add(file_name)
- return existing_files
-
- def _get_expected_files(self, repre):
- """Returns set of file names from metadata.json"""
- expected_files = set()
-
- files = repre["files"]
- if not isinstance(files, list):
- files = [files]
-
- for file_name in files:
- expected_files.add(file_name)
- return expected_files
diff --git a/openpype/modules/default_modules/ftrack/__init__.py b/openpype/modules/default_modules/ftrack/__init__.py
deleted file mode 100644
index 7261254c6fe..00000000000
--- a/openpype/modules/default_modules/ftrack/__init__.py
+++ /dev/null
@@ -1,9 +0,0 @@
-from .ftrack_module import (
- FtrackModule,
- FTRACK_MODULE_DIR
-)
-
-__all__ = (
- "FtrackModule",
- "FTRACK_MODULE_DIR"
-)
diff --git a/openpype/modules/default_modules/ftrack/event_handlers_server/action_prepare_project.py b/openpype/modules/default_modules/ftrack/event_handlers_server/action_prepare_project.py
deleted file mode 100644
index 2e55be27438..00000000000
--- a/openpype/modules/default_modules/ftrack/event_handlers_server/action_prepare_project.py
+++ /dev/null
@@ -1,414 +0,0 @@
-import json
-
-from avalon.api import AvalonMongoDB
-from openpype.api import ProjectSettings
-from openpype.lib import create_project
-from openpype.settings import SaveWarningExc
-
-from openpype_modules.ftrack.lib import (
- ServerAction,
- get_openpype_attr,
- CUST_ATTR_AUTO_SYNC
-)
-
-
-class PrepareProjectServer(ServerAction):
- """Prepare project attributes in Anatomy."""
-
- identifier = "prepare.project.server"
- label = "OpenPype Admin"
- variant = "- Prepare Project (Server)"
- description = "Set basic attributes on the project"
-
- settings_key = "prepare_project"
-
- role_list = ["Pypeclub", "Administrator", "Project Manager"]
-
- settings_key = "prepare_project"
-
- item_splitter = {"type": "label", "value": "---"}
- _keys_order = (
- "fps",
- "frameStart",
- "frameEnd",
- "handleStart",
- "handleEnd",
- "clipIn",
- "clipOut",
- "resolutionHeight",
- "resolutionWidth",
- "pixelAspect",
- "applications",
- "tools_env",
- "library_project",
- )
-
- def discover(self, session, entities, event):
- """Show only on project."""
- if (
- len(entities) != 1
- or entities[0].entity_type.lower() != "project"
- ):
- return False
-
- return self.valid_roles(session, entities, event)
-
- def interface(self, session, entities, event):
- if event['data'].get('values', {}):
- return
-
- # Inform user that this may take a while
- self.show_message(event, "Preparing data... Please wait", True)
- self.log.debug("Preparing data which will be shown")
-
- self.log.debug("Loading custom attributes")
-
- project_entity = entities[0]
- project_name = project_entity["full_name"]
-
- project_settings = ProjectSettings(project_name)
-
- project_anatom_settings = project_settings["project_anatomy"]
- root_items = self.prepare_root_items(project_anatom_settings)
-
- ca_items, multiselect_enumerators = (
- self.prepare_custom_attribute_items(project_anatom_settings)
- )
-
- self.log.debug("Heavy items are ready. Preparing last items group.")
-
- title = "Prepare Project"
- items = []
-
- # Add root items
- items.extend(root_items)
-
- items.append(self.item_splitter)
- items.append({
- "type": "label",
- "value": "Set basic Attributes: "
- })
-
- items.extend(ca_items)
-
- # This item will be last before enumerators
- # Set value of auto synchronization
- auto_sync_value = project_entity["custom_attributes"].get(
- CUST_ATTR_AUTO_SYNC, False
- )
- auto_sync_item = {
- "name": CUST_ATTR_AUTO_SYNC,
- "type": "boolean",
- "value": auto_sync_value,
- "label": "AutoSync to Avalon"
- }
- # Add autosync attribute
- items.append(auto_sync_item)
-
- # Add enumerator items at the end
- for item in multiselect_enumerators:
- items.append(item)
-
- return {
- "items": items,
- "title": title
- }
-
- def prepare_root_items(self, project_anatom_settings):
- self.log.debug("Root items preparation begins.")
-
- root_items = []
- root_items.append({
- "type": "label",
- "value": "Check your Project root settings "
- })
- root_items.append({
- "type": "label",
- "value": (
- "NOTE: Roots are crutial for path filling"
- " (and creating folder structure).
"
- )
- })
- root_items.append({
- "type": "label",
- "value": (
- "WARNING: Do not change roots on running project,"
- " that will cause workflow issues .
"
- )
- })
-
- empty_text = "Enter root path here..."
-
- roots_entity = project_anatom_settings["roots"]
- for root_name, root_entity in roots_entity.items():
- root_items.append(self.item_splitter)
- root_items.append({
- "type": "label",
- "value": "Root: \"{}\"".format(root_name)
- })
- for platform_name, value_entity in root_entity.items():
- root_items.append({
- "label": platform_name,
- "name": "__root__{}__{}".format(root_name, platform_name),
- "type": "text",
- "value": value_entity.value,
- "empty_text": empty_text
- })
-
- root_items.append({
- "type": "hidden",
- "name": "__rootnames__",
- "value": json.dumps(list(roots_entity.keys()))
- })
-
- self.log.debug("Root items preparation ended.")
- return root_items
-
- def _attributes_to_set(self, project_anatom_settings):
- attributes_to_set = {}
-
- attribute_values_by_key = {}
- for key, entity in project_anatom_settings["attributes"].items():
- attribute_values_by_key[key] = entity.value
-
- cust_attrs, hier_cust_attrs = get_openpype_attr(self.session, True)
-
- for attr in hier_cust_attrs:
- key = attr["key"]
- if key.startswith("avalon_"):
- continue
- attributes_to_set[key] = {
- "label": attr["label"],
- "object": attr,
- "default": attribute_values_by_key.get(key)
- }
-
- for attr in cust_attrs:
- if attr["entity_type"].lower() != "show":
- continue
- key = attr["key"]
- if key.startswith("avalon_"):
- continue
- attributes_to_set[key] = {
- "label": attr["label"],
- "object": attr,
- "default": attribute_values_by_key.get(key)
- }
-
- # Sort by label
- attributes_to_set = dict(sorted(
- attributes_to_set.items(),
- key=lambda x: x[1]["label"]
- ))
- return attributes_to_set
-
- def prepare_custom_attribute_items(self, project_anatom_settings):
- items = []
- multiselect_enumerators = []
- attributes_to_set = self._attributes_to_set(project_anatom_settings)
-
- self.log.debug("Preparing interface for keys: \"{}\"".format(
- str([key for key in attributes_to_set])
- ))
-
- attribute_keys = set(attributes_to_set.keys())
- keys_order = []
- for key in self._keys_order:
- if key in attribute_keys:
- keys_order.append(key)
-
- attribute_keys = attribute_keys - set(keys_order)
- for key in sorted(attribute_keys):
- keys_order.append(key)
-
- for key in keys_order:
- in_data = attributes_to_set[key]
- attr = in_data["object"]
-
- # initial item definition
- item = {
- "name": key,
- "label": in_data["label"]
- }
-
- # cust attr type - may have different visualization
- type_name = attr["type"]["name"].lower()
- easy_types = ["text", "boolean", "date", "number"]
-
- easy_type = False
- if type_name in easy_types:
- easy_type = True
-
- elif type_name == "enumerator":
-
- attr_config = json.loads(attr["config"])
- attr_config_data = json.loads(attr_config["data"])
-
- if attr_config["multiSelect"] is True:
- multiselect_enumerators.append(self.item_splitter)
- multiselect_enumerators.append({
- "type": "label",
- "value": "{} ".format(in_data["label"])
- })
-
- default = in_data["default"]
- names = []
- for option in sorted(
- attr_config_data, key=lambda x: x["menu"]
- ):
- name = option["value"]
- new_name = "__{}__{}".format(key, name)
- names.append(new_name)
- item = {
- "name": new_name,
- "type": "boolean",
- "label": "- {}".format(option["menu"])
- }
- if default:
- if isinstance(default, (list, tuple)):
- if name in default:
- item["value"] = True
- else:
- if name == default:
- item["value"] = True
-
- multiselect_enumerators.append(item)
-
- multiselect_enumerators.append({
- "type": "hidden",
- "name": "__hidden__{}".format(key),
- "value": json.dumps(names)
- })
- else:
- easy_type = True
- item["data"] = attr_config_data
-
- else:
- self.log.warning((
- "Custom attribute \"{}\" has type \"{}\"."
- " I don't know how to handle"
- ).format(key, type_name))
- items.append({
- "type": "label",
- "value": (
- "!!! Can't handle Custom attritubte type \"{}\""
- " (key: \"{}\")"
- ).format(type_name, key)
- })
-
- if easy_type:
- item["type"] = type_name
-
- # default value in interface
- default = in_data["default"]
- if default is not None:
- item["value"] = default
-
- items.append(item)
-
- return items, multiselect_enumerators
-
- def launch(self, session, entities, event):
- in_data = event["data"].get("values")
- if not in_data:
- return
-
- root_values = {}
- root_key = "__root__"
- for key in tuple(in_data.keys()):
- if key.startswith(root_key):
- _key = key[len(root_key):]
- root_values[_key] = in_data.pop(key)
-
- root_names = in_data.pop("__rootnames__", None)
- root_data = {}
- for root_name in json.loads(root_names):
- root_data[root_name] = {}
- for key, value in tuple(root_values.items()):
- prefix = "{}__".format(root_name)
- if not key.startswith(prefix):
- continue
-
- _key = key[len(prefix):]
- root_data[root_name][_key] = value
-
- # Find hidden items for multiselect enumerators
- keys_to_process = []
- for key in in_data:
- if key.startswith("__hidden__"):
- keys_to_process.append(key)
-
- self.log.debug("Preparing data for Multiselect Enumerators")
- enumerators = {}
- for key in keys_to_process:
- new_key = key.replace("__hidden__", "")
- enumerator_items = in_data.pop(key)
- enumerators[new_key] = json.loads(enumerator_items)
-
- # find values set for multiselect enumerator
- for key, enumerator_items in enumerators.items():
- in_data[key] = []
-
- name = "__{}__".format(key)
-
- for item in enumerator_items:
- value = in_data.pop(item)
- if value is True:
- new_key = item.replace(name, "")
- in_data[key].append(new_key)
-
- self.log.debug("Setting Custom Attribute values")
-
- project_entity = entities[0]
- project_name = project_entity["full_name"]
-
- # Try to find project document
- dbcon = AvalonMongoDB()
- dbcon.install()
- dbcon.Session["AVALON_PROJECT"] = project_name
- project_doc = dbcon.find_one({
- "type": "project"
- })
- # Create project if is not available
- # - creation is required to be able set project anatomy and attributes
- if not project_doc:
- project_code = project_entity["name"]
- self.log.info("Creating project \"{} [{}]\"".format(
- project_name, project_code
- ))
- create_project(project_name, project_code, dbcon=dbcon)
-
- dbcon.uninstall()
-
- project_settings = ProjectSettings(project_name)
- project_anatomy_settings = project_settings["project_anatomy"]
- project_anatomy_settings["roots"] = root_data
-
- custom_attribute_values = {}
- attributes_entity = project_anatomy_settings["attributes"]
- for key, value in in_data.items():
- if key not in attributes_entity:
- custom_attribute_values[key] = value
- else:
- attributes_entity[key] = value
-
- try:
- project_settings.save()
- except SaveWarningExc as exc:
- self.log.info("Few warnings happened during settings save:")
- for warning in exc.warnings:
- self.log.info(str(warning))
-
- # Change custom attributes on project
- if custom_attribute_values:
- for key, value in custom_attribute_values.items():
- project_entity["custom_attributes"][key] = value
- self.log.debug("- Key \"{}\" set to \"{}\"".format(key, value))
- session.commit()
-
- return True
-
-
-def register(session):
- '''Register plugin. Called when used as an plugin.'''
- PrepareProjectServer(session).register()
diff --git a/openpype/modules/default_modules/ftrack/event_handlers_server/action_sync_to_avalon.py b/openpype/modules/default_modules/ftrack/event_handlers_server/action_sync_to_avalon.py
deleted file mode 100644
index 58f79e8a2be..00000000000
--- a/openpype/modules/default_modules/ftrack/event_handlers_server/action_sync_to_avalon.py
+++ /dev/null
@@ -1,212 +0,0 @@
-import time
-import sys
-import json
-import traceback
-
-from openpype_modules.ftrack.lib import ServerAction
-from openpype_modules.ftrack.lib.avalon_sync import SyncEntitiesFactory
-
-
-class SyncToAvalonServer(ServerAction):
- """
- Synchronizing data action - from Ftrack to Avalon DB
-
- Stores all information about entity.
- - Name(string) - Most important information = identifier of entity
- - Parent(ObjectId) - Avalon Project Id, if entity is not project itself
- - Data(dictionary):
- - VisualParent(ObjectId) - Avalon Id of parent asset
- - Parents(array of string) - All parent names except project
- - Tasks(dictionary of dictionaries) - Tasks on asset
- - FtrackId(string)
- - entityType(string) - entity's type on Ftrack
- * All Custom attributes in group 'Avalon'
- - custom attributes that start with 'avalon_' are skipped
-
- * These information are stored for entities in whole project.
-
- Avalon ID of asset is stored to Ftrack
- - Custom attribute 'avalon_mongo_id'.
- - action IS NOT creating this Custom attribute if doesn't exist
- - run 'Create Custom Attributes' action
- - or do it manually (Not recommended)
- """
- #: Action identifier.
- identifier = "sync.to.avalon.server"
- #: Action label.
- label = "OpenPype Admin"
- variant = "- Sync To Avalon (Server)"
- #: Action description.
- description = "Send data from Ftrack to Avalon"
- role_list = {"Pypeclub", "Administrator", "Project Manager"}
-
- def __init__(self, *args, **kwargs):
- super().__init__(*args, **kwargs)
- self.entities_factory = SyncEntitiesFactory(self.log, self.session)
-
- def discover(self, session, entities, event):
- """ Validation """
- # Check if selection is valid
- for ent in event["data"]["selection"]:
- # Ignore entities that are not tasks or projects
- if ent["entityType"].lower() in ["show", "task"]:
- return True
- return False
-
- def launch(self, session, in_entities, event):
- self.log.debug("{}: Creating job".format(self.label))
-
- user_entity = session.query(
- "User where id is {}".format(event["source"]["user"]["id"])
- ).one()
- job_entity = session.create("Job", {
- "user": user_entity,
- "status": "running",
- "data": json.dumps({
- "description": "Sync to avalon is running..."
- })
- })
- session.commit()
-
- project_entity = self.get_project_from_entity(in_entities[0])
- project_name = project_entity["full_name"]
-
- try:
- result = self.synchronization(event, project_name)
-
- except Exception:
- self.log.error(
- "Synchronization failed due to code error", exc_info=True
- )
-
- description = "Sync to avalon Crashed (Download traceback)"
- self.add_traceback_to_job(
- job_entity, session, sys.exc_info(), description
- )
-
- msg = "An error has happened during synchronization"
- title = "Synchronization report ({}):".format(project_name)
- items = []
- items.append({
- "type": "label",
- "value": "# {}".format(msg)
- })
- items.append({
- "type": "label",
- "value": (
- "Download report from job for more information.
"
- )
- })
-
- report = {}
- try:
- report = self.entities_factory.report()
- except Exception:
- pass
-
- _items = report.get("items") or []
- if _items:
- items.append(self.entities_factory.report_splitter)
- items.extend(_items)
-
- self.show_interface(items, title, event, submit_btn_label="Ok")
-
- return {"success": True, "message": msg}
-
- job_entity["status"] = "done"
- job_entity["data"] = json.dumps({
- "description": "Sync to avalon finished."
- })
- session.commit()
-
- return result
-
- def synchronization(self, event, project_name):
- time_start = time.time()
-
- self.show_message(event, "Synchronization - Preparing data", True)
-
- try:
- output = self.entities_factory.launch_setup(project_name)
- if output is not None:
- return output
-
- time_1 = time.time()
-
- self.entities_factory.set_cutom_attributes()
- time_2 = time.time()
-
- # This must happen before all filtering!!!
- self.entities_factory.prepare_avalon_entities(project_name)
- time_3 = time.time()
-
- self.entities_factory.filter_by_ignore_sync()
- time_4 = time.time()
-
- self.entities_factory.duplicity_regex_check()
- time_5 = time.time()
-
- self.entities_factory.prepare_ftrack_ent_data()
- time_6 = time.time()
-
- self.entities_factory.synchronize()
- time_7 = time.time()
-
- self.log.debug(
- "*** Synchronization finished ***"
- )
- self.log.debug(
- "preparation <{}>".format(time_1 - time_start)
- )
- self.log.debug(
- "set_cutom_attributes <{}>".format(time_2 - time_1)
- )
- self.log.debug(
- "prepare_avalon_entities <{}>".format(time_3 - time_2)
- )
- self.log.debug(
- "filter_by_ignore_sync <{}>".format(time_4 - time_3)
- )
- self.log.debug(
- "duplicity_regex_check <{}>".format(time_5 - time_4)
- )
- self.log.debug(
- "prepare_ftrack_ent_data <{}>".format(time_6 - time_5)
- )
- self.log.debug(
- "synchronize <{}>".format(time_7 - time_6)
- )
- self.log.debug(
- "* Total time: {}".format(time_7 - time_start)
- )
-
- report = self.entities_factory.report()
- if report and report.get("items"):
- default_title = "Synchronization report ({}):".format(
- project_name
- )
- self.show_interface(
- items=report["items"],
- title=report.get("title", default_title),
- event=event
- )
- return {
- "success": True,
- "message": "Synchronization Finished"
- }
-
- finally:
- try:
- self.entities_factory.dbcon.uninstall()
- except Exception:
- pass
-
- try:
- self.entities_factory.session.close()
- except Exception:
- pass
-
-
-def register(session):
- '''Register plugin. Called when used as an plugin.'''
- SyncToAvalonServer(session).register()
diff --git a/openpype/modules/default_modules/ftrack/event_handlers_user/action_create_folders.py b/openpype/modules/default_modules/ftrack/event_handlers_user/action_create_folders.py
deleted file mode 100644
index 994dbd90e48..00000000000
--- a/openpype/modules/default_modules/ftrack/event_handlers_user/action_create_folders.py
+++ /dev/null
@@ -1,248 +0,0 @@
-import os
-from openpype_modules.ftrack.lib import BaseAction, statics_icon
-from avalon import lib as avalonlib
-from openpype.api import (
- Anatomy,
- get_project_settings
-)
-from openpype.lib import ApplicationManager
-
-
-class CreateFolders(BaseAction):
- identifier = "create.folders"
- label = "Create Folders"
- icon = statics_icon("ftrack", "action_icons", "CreateFolders.svg")
-
- def discover(self, session, entities, event):
- if len(entities) != 1:
- return False
-
- not_allowed = ["assetversion", "project"]
- if entities[0].entity_type.lower() in not_allowed:
- return False
-
- return True
-
- def interface(self, session, entities, event):
- if event["data"].get("values", {}):
- return
- entity = entities[0]
- without_interface = True
- for child in entity["children"]:
- if child["object_type"]["name"].lower() != "task":
- without_interface = False
- break
- self.without_interface = without_interface
- if without_interface:
- return
- title = "Create folders"
-
- entity_name = entity["name"]
- msg = (
- "Do you want create folders also"
- " for all children of \"{}\"? "
- )
- if entity.entity_type.lower() == "project":
- entity_name = entity["full_name"]
- msg = msg.replace(" also", "")
- msg += "(Project root won't be created if not checked) "
- items = []
- item_msg = {
- "type": "label",
- "value": msg.format(entity_name)
- }
- item_label = {
- "type": "label",
- "value": "With all chilren entities"
- }
- item = {
- "name": "children_included",
- "type": "boolean",
- "value": False
- }
- items.append(item_msg)
- items.append(item_label)
- items.append(item)
-
- return {
- "items": items,
- "title": title
- }
-
- def launch(self, session, entities, event):
- '''Callback method for custom action.'''
- with_childrens = True
- if self.without_interface is False:
- if "values" not in event["data"]:
- return
- with_childrens = event["data"]["values"]["children_included"]
-
- entity = entities[0]
- if entity.entity_type.lower() == "project":
- proj = entity
- else:
- proj = entity["project"]
- project_name = proj["full_name"]
- project_code = proj["name"]
-
- if entity.entity_type.lower() == 'project' and with_childrens is False:
- return {
- 'success': True,
- 'message': 'Nothing was created'
- }
-
- all_entities = []
- all_entities.append(entity)
- if with_childrens:
- all_entities = self.get_notask_children(entity)
-
- anatomy = Anatomy(project_name)
- project_settings = get_project_settings(project_name)
-
- work_keys = ["work", "folder"]
- work_template = anatomy.templates
- for key in work_keys:
- work_template = work_template[key]
- work_has_apps = "{app" in work_template
-
- publish_keys = ["publish", "folder"]
- publish_template = anatomy.templates
- for key in publish_keys:
- publish_template = publish_template[key]
- publish_has_apps = "{app" in publish_template
-
- tools_settings = project_settings["global"]["tools"]
- app_presets = tools_settings["Workfiles"]["sw_folders"]
- app_manager_apps = None
- if app_presets and (work_has_apps or publish_has_apps):
- app_manager_apps = ApplicationManager().applications
-
- cached_apps = {}
- collected_paths = []
- for entity in all_entities:
- if entity.entity_type.lower() == "project":
- continue
- ent_data = {
- "project": {
- "name": project_name,
- "code": project_code
- }
- }
-
- ent_data["asset"] = entity["name"]
-
- parents = entity["link"][1:-1]
- hierarchy_names = [p["name"] for p in parents]
- hierarchy = ""
- if hierarchy_names:
- hierarchy = os.path.sep.join(hierarchy_names)
- ent_data["hierarchy"] = hierarchy
-
- tasks_created = False
- for child in entity["children"]:
- if child["object_type"]["name"].lower() != "task":
- continue
- tasks_created = True
- task_type_name = child["type"]["name"].lower()
- task_data = ent_data.copy()
- task_data["task"] = child["name"]
-
- apps = []
- if app_manager_apps:
- possible_apps = app_presets.get(task_type_name) or []
- for app_name in possible_apps:
-
- if app_name in cached_apps:
- apps.append(cached_apps[app_name])
- continue
-
- app_def = app_manager_apps.get(app_name)
- if app_def and app_def.is_host:
- app_dir = app_def.host_name
- else:
- app_dir = app_name
- cached_apps[app_name] = app_dir
- apps.append(app_dir)
-
- # Template wok
- if work_has_apps:
- app_data = task_data.copy()
- for app in apps:
- app_data["app"] = app
- collected_paths.append(self.compute_template(
- anatomy, app_data, work_keys
- ))
- else:
- collected_paths.append(self.compute_template(
- anatomy, task_data, work_keys
- ))
-
- # Template publish
- if publish_has_apps:
- app_data = task_data.copy()
- for app in apps:
- app_data["app"] = app
- collected_paths.append(self.compute_template(
- anatomy, app_data, publish_keys
- ))
- else:
- collected_paths.append(self.compute_template(
- anatomy, task_data, publish_keys
- ))
-
- if not tasks_created:
- # create path for entity
- collected_paths.append(self.compute_template(
- anatomy, ent_data, work_keys
- ))
- collected_paths.append(self.compute_template(
- anatomy, ent_data, publish_keys
- ))
-
- if len(collected_paths) == 0:
- return {
- "success": True,
- "message": "No project folders to create."
- }
-
- self.log.info("Creating folders:")
-
- for path in set(collected_paths):
- self.log.info(path)
- if not os.path.exists(path):
- os.makedirs(path)
-
- return {
- "success": True,
- "message": "Successfully created project folders."
- }
-
- def get_notask_children(self, entity):
- output = []
- if entity.entity_type.lower() == "task":
- return output
-
- output.append(entity)
- for child in entity["children"]:
- output.extend(self.get_notask_children(child))
- return output
-
- def compute_template(self, anatomy, data, anatomy_keys):
- filled_template = anatomy.format_all(data)
- for key in anatomy_keys:
- filled_template = filled_template[key]
-
- if filled_template.solved:
- return os.path.normpath(filled_template)
-
- self.log.warning(
- "Template \"{}\" was not fully filled \"{}\"".format(
- filled_template.template, filled_template
- )
- )
- return os.path.normpath(filled_template.split("{")[0])
-
-
-def register(session):
- """Register plugin. Called when used as an plugin."""
- CreateFolders(session).register()
diff --git a/openpype/modules/default_modules/ftrack/event_handlers_user/action_delivery.py b/openpype/modules/default_modules/ftrack/event_handlers_user/action_delivery.py
deleted file mode 100644
index 1f28b189007..00000000000
--- a/openpype/modules/default_modules/ftrack/event_handlers_user/action_delivery.py
+++ /dev/null
@@ -1,568 +0,0 @@
-import os
-import copy
-import json
-import collections
-
-from bson.objectid import ObjectId
-
-from openpype.api import Anatomy, config
-from openpype_modules.ftrack.lib import BaseAction, statics_icon
-from openpype_modules.ftrack.lib.avalon_sync import CUST_ATTR_ID_KEY
-from openpype.lib.delivery import (
- path_from_representation,
- get_format_dict,
- check_destination_path,
- process_single_file,
- process_sequence
-)
-from avalon.api import AvalonMongoDB
-
-
-class Delivery(BaseAction):
-
- identifier = "delivery.action"
- label = "Delivery"
- description = "Deliver data to client"
- role_list = ["Pypeclub", "Administrator", "Project manager"]
- icon = statics_icon("ftrack", "action_icons", "Delivery.svg")
- settings_key = "delivery_action"
-
- def __init__(self, *args, **kwargs):
- self.db_con = AvalonMongoDB()
-
- super(Delivery, self).__init__(*args, **kwargs)
-
- def discover(self, session, entities, event):
- is_valid = False
- for entity in entities:
- if entity.entity_type.lower() == "assetversion":
- is_valid = True
- break
-
- if is_valid:
- is_valid = self.valid_roles(session, entities, event)
- return is_valid
-
- def interface(self, session, entities, event):
- if event["data"].get("values", {}):
- return
-
- title = "Delivery data to Client"
-
- items = []
- item_splitter = {"type": "label", "value": "---"}
-
- project_entity = self.get_project_from_entity(entities[0])
- project_name = project_entity["full_name"]
- self.db_con.install()
- self.db_con.Session["AVALON_PROJECT"] = project_name
- project_doc = self.db_con.find_one({"type": "project"})
- if not project_doc:
- return {
- "success": False,
- "message": (
- "Didn't found project \"{}\" in avalon."
- ).format(project_name)
- }
-
- repre_names = self._get_repre_names(entities)
- self.db_con.uninstall()
-
- items.append({
- "type": "hidden",
- "name": "__project_name__",
- "value": project_name
- })
-
- # Prepare anatomy data
- anatomy = Anatomy(project_name)
- new_anatomies = []
- first = None
- for key, template in (anatomy.templates.get("delivery") or {}).items():
- # Use only keys with `{root}` or `{root[*]}` in value
- if isinstance(template, str) and "{root" in template:
- new_anatomies.append({
- "label": key,
- "value": key
- })
- if first is None:
- first = key
-
- skipped = False
- # Add message if there are any common components
- if not repre_names or not new_anatomies:
- skipped = True
- items.append({
- "type": "label",
- "value": "Something went wrong: "
- })
-
- items.append({
- "type": "hidden",
- "name": "__skipped__",
- "value": skipped
- })
-
- if not repre_names:
- if len(entities) == 1:
- items.append({
- "type": "label",
- "value": (
- "- Selected entity doesn't have components to deliver."
- )
- })
- else:
- items.append({
- "type": "label",
- "value": (
- "- Selected entities don't have common components."
- )
- })
-
- # Add message if delivery anatomies are not set
- if not new_anatomies:
- items.append({
- "type": "label",
- "value": (
- "- `\"delivery\"` anatomy key is not set in config."
- )
- })
-
- # Skip if there are any data shortcomings
- if skipped:
- return {
- "items": items,
- "title": title
- }
-
- items.append({
- "value": "Choose Components to deliver ",
- "type": "label"
- })
-
- for repre_name in repre_names:
- items.append({
- "type": "boolean",
- "value": False,
- "label": repre_name,
- "name": repre_name
- })
-
- items.append(item_splitter)
-
- items.append({
- "value": "Location for delivery ",
- "type": "label"
- })
-
- items.append({
- "type": "label",
- "value": (
- "NOTE: It is possible to replace `root` key in anatomy. "
- )
- })
-
- items.append({
- "type": "text",
- "name": "__location_path__",
- "empty_text": "Type location path here...(Optional)"
- })
-
- items.append(item_splitter)
-
- items.append({
- "value": "Anatomy of delivery files ",
- "type": "label"
- })
-
- items.append({
- "type": "label",
- "value": (
- "NOTE: These can be set in Anatomy.yaml"
- " within `delivery` key.
"
- )
- })
-
- items.append({
- "type": "enumerator",
- "name": "__new_anatomies__",
- "data": new_anatomies,
- "value": first
- })
-
- return {
- "items": items,
- "title": title
- }
-
- def _get_repre_names(self, entities):
- version_ids = self._get_interest_version_ids(entities)
- repre_docs = self.db_con.find({
- "type": "representation",
- "parent": {"$in": version_ids}
- })
- return list(sorted(repre_docs.distinct("name")))
-
- def _get_interest_version_ids(self, entities):
- parent_ent_by_id = {}
- subset_names = set()
- version_nums = set()
- for entity in entities:
- asset = entity["asset"]
- parent = asset["parent"]
- parent_ent_by_id[parent["id"]] = parent
-
- subset_name = asset["name"]
- subset_names.add(subset_name)
-
- version = entity["version"]
- version_nums.add(version)
-
- asset_docs_by_ftrack_id = self._get_asset_docs(parent_ent_by_id)
- subset_docs = self._get_subset_docs(
- asset_docs_by_ftrack_id, subset_names, entities
- )
- version_docs = self._get_version_docs(
- asset_docs_by_ftrack_id, subset_docs, version_nums, entities
- )
-
- return [version_doc["_id"] for version_doc in version_docs]
-
- def _get_version_docs(
- self, asset_docs_by_ftrack_id, subset_docs, version_nums, entities
- ):
- subset_docs_by_id = {
- subset_doc["_id"]: subset_doc
- for subset_doc in subset_docs
- }
- version_docs = list(self.db_con.find({
- "type": "version",
- "parent": {"$in": list(subset_docs_by_id.keys())},
- "name": {"$in": list(version_nums)}
- }))
- version_docs_by_parent_id = collections.defaultdict(dict)
- for version_doc in version_docs:
- subset_doc = subset_docs_by_id[version_doc["parent"]]
-
- asset_id = subset_doc["parent"]
- subset_name = subset_doc["name"]
- version = version_doc["name"]
- if version_docs_by_parent_id[asset_id].get(subset_name) is None:
- version_docs_by_parent_id[asset_id][subset_name] = {}
-
- version_docs_by_parent_id[asset_id][subset_name][version] = (
- version_doc
- )
-
- filtered_versions = []
- for entity in entities:
- asset = entity["asset"]
-
- parent = asset["parent"]
- asset_doc = asset_docs_by_ftrack_id[parent["id"]]
-
- subsets_by_name = version_docs_by_parent_id.get(asset_doc["_id"])
- if not subsets_by_name:
- continue
-
- subset_name = asset["name"]
- version_docs_by_version = subsets_by_name.get(subset_name)
- if not version_docs_by_version:
- continue
-
- version = entity["version"]
- version_doc = version_docs_by_version.get(version)
- if version_doc:
- filtered_versions.append(version_doc)
- return filtered_versions
-
- def _get_subset_docs(
- self, asset_docs_by_ftrack_id, subset_names, entities
- ):
- asset_doc_ids = list()
- for asset_doc in asset_docs_by_ftrack_id.values():
- asset_doc_ids.append(asset_doc["_id"])
-
- subset_docs = list(self.db_con.find({
- "type": "subset",
- "parent": {"$in": asset_doc_ids},
- "name": {"$in": list(subset_names)}
- }))
- subset_docs_by_parent_id = collections.defaultdict(dict)
- for subset_doc in subset_docs:
- asset_id = subset_doc["parent"]
- subset_name = subset_doc["name"]
- subset_docs_by_parent_id[asset_id][subset_name] = subset_doc
-
- filtered_subsets = []
- for entity in entities:
- asset = entity["asset"]
-
- parent = asset["parent"]
- asset_doc = asset_docs_by_ftrack_id[parent["id"]]
-
- subsets_by_name = subset_docs_by_parent_id.get(asset_doc["_id"])
- if not subsets_by_name:
- continue
-
- subset_name = asset["name"]
- subset_doc = subsets_by_name.get(subset_name)
- if subset_doc:
- filtered_subsets.append(subset_doc)
- return filtered_subsets
-
- def _get_asset_docs(self, parent_ent_by_id):
- asset_docs = list(self.db_con.find({
- "type": "asset",
- "data.ftrackId": {"$in": list(parent_ent_by_id.keys())}
- }))
- asset_docs_by_ftrack_id = {
- asset_doc["data"]["ftrackId"]: asset_doc
- for asset_doc in asset_docs
- }
-
- entities_by_mongo_id = {}
- entities_by_names = {}
- for ftrack_id, entity in parent_ent_by_id.items():
- if ftrack_id not in asset_docs_by_ftrack_id:
- parent_mongo_id = entity["custom_attributes"].get(
- CUST_ATTR_ID_KEY
- )
- if parent_mongo_id:
- entities_by_mongo_id[ObjectId(parent_mongo_id)] = entity
- else:
- entities_by_names[entity["name"]] = entity
-
- expressions = []
- if entities_by_mongo_id:
- expression = {
- "type": "asset",
- "_id": {"$in": list(entities_by_mongo_id.keys())}
- }
- expressions.append(expression)
-
- if entities_by_names:
- expression = {
- "type": "asset",
- "name": {"$in": list(entities_by_names.keys())}
- }
- expressions.append(expression)
-
- if expressions:
- if len(expressions) == 1:
- filter = expressions[0]
- else:
- filter = {"$or": expressions}
-
- asset_docs = self.db_con.find(filter)
- for asset_doc in asset_docs:
- if asset_doc["_id"] in entities_by_mongo_id:
- entity = entities_by_mongo_id[asset_doc["_id"]]
- asset_docs_by_ftrack_id[entity["id"]] = asset_doc
-
- elif asset_doc["name"] in entities_by_names:
- entity = entities_by_names[asset_doc["name"]]
- asset_docs_by_ftrack_id[entity["id"]] = asset_doc
-
- return asset_docs_by_ftrack_id
-
- def launch(self, session, entities, event):
- if "values" not in event["data"]:
- return {
- "success": True,
- "message": "Nothing to do"
- }
-
- values = event["data"]["values"]
- skipped = values.pop("__skipped__")
- if skipped:
- return {
- "success": False,
- "message": "Action skipped"
- }
-
- user_id = event["source"]["user"]["id"]
- user_entity = session.query(
- "User where id is {}".format(user_id)
- ).one()
-
- job = session.create("Job", {
- "user": user_entity,
- "status": "running",
- "data": json.dumps({
- "description": "Delivery processing."
- })
- })
- session.commit()
-
- try:
- self.db_con.install()
- report = self.real_launch(session, entities, event)
-
- except Exception as exc:
- report = {
- "success": False,
- "title": "Delivery failed",
- "items": [{
- "type": "label",
- "value": (
- "Error during delivery action process:{}
'.format("Set basic Attributes: "
- })
-
- items.extend(ca_items)
-
- # Set value of auto synchronization
- auto_sync_value = project_entity["custom_attributes"].get(
- CUST_ATTR_AUTO_SYNC, False
- )
- auto_sync_item = {
- "name": CUST_ATTR_AUTO_SYNC,
- "type": "boolean",
- "value": auto_sync_value,
- "label": "AutoSync to Avalon"
- }
- # Add autosync attribute
- items.append(auto_sync_item)
-
- # This item will be last before enumerators
- # Ask if want to trigger Action Create Folder Structure
- create_project_structure_checked = (
- project_settings
- ["project_settings"]
- ["ftrack"]
- ["user_handlers"]
- ["prepare_project"]
- ["create_project_structure_checked"]
- ).value
- items.append({
- "type": "label",
- "value": "Want to create basic Folder Structure? "
- })
- items.append({
- "name": self.create_project_structure_key,
- "type": "boolean",
- "value": create_project_structure_checked,
- "label": "Check if Yes"
- })
-
- # Add enumerator items at the end
- for item in multiselect_enumerators:
- items.append(item)
-
- return {
- "items": items,
- "title": title
- }
-
- def prepare_root_items(self, project_anatom_settings):
- self.log.debug("Root items preparation begins.")
-
- root_items = []
- root_items.append({
- "type": "label",
- "value": "Check your Project root settings "
- })
- root_items.append({
- "type": "label",
- "value": (
- "NOTE: Roots are crutial for path filling"
- " (and creating folder structure).
"
- )
- })
- root_items.append({
- "type": "label",
- "value": (
- "WARNING: Do not change roots on running project,"
- " that will cause workflow issues .
"
- )
- })
-
- empty_text = "Enter root path here..."
-
- roots_entity = project_anatom_settings["roots"]
- for root_name, root_entity in roots_entity.items():
- root_items.append(self.item_splitter)
- root_items.append({
- "type": "label",
- "value": "Root: \"{}\"".format(root_name)
- })
- for platform_name, value_entity in root_entity.items():
- root_items.append({
- "label": platform_name,
- "name": "__root__{}__{}".format(root_name, platform_name),
- "type": "text",
- "value": value_entity.value,
- "empty_text": empty_text
- })
-
- root_items.append({
- "type": "hidden",
- "name": "__rootnames__",
- "value": json.dumps(list(roots_entity.keys()))
- })
-
- self.log.debug("Root items preparation ended.")
- return root_items
-
- def _attributes_to_set(self, project_anatom_settings):
- attributes_to_set = {}
-
- attribute_values_by_key = {}
- for key, entity in project_anatom_settings["attributes"].items():
- attribute_values_by_key[key] = entity.value
-
- cust_attrs, hier_cust_attrs = get_openpype_attr(self.session, True)
-
- for attr in hier_cust_attrs:
- key = attr["key"]
- if key.startswith("avalon_"):
- continue
- attributes_to_set[key] = {
- "label": attr["label"],
- "object": attr,
- "default": attribute_values_by_key.get(key)
- }
-
- for attr in cust_attrs:
- if attr["entity_type"].lower() != "show":
- continue
- key = attr["key"]
- if key.startswith("avalon_"):
- continue
- attributes_to_set[key] = {
- "label": attr["label"],
- "object": attr,
- "default": attribute_values_by_key.get(key)
- }
-
- # Sort by label
- attributes_to_set = dict(sorted(
- attributes_to_set.items(),
- key=lambda x: x[1]["label"]
- ))
- return attributes_to_set
-
- def prepare_custom_attribute_items(self, project_anatom_settings):
- items = []
- multiselect_enumerators = []
- attributes_to_set = self._attributes_to_set(project_anatom_settings)
-
- self.log.debug("Preparing interface for keys: \"{}\"".format(
- str([key for key in attributes_to_set])
- ))
-
- attribute_keys = set(attributes_to_set.keys())
- keys_order = []
- for key in self._keys_order:
- if key in attribute_keys:
- keys_order.append(key)
-
- attribute_keys = attribute_keys - set(keys_order)
- for key in sorted(attribute_keys):
- keys_order.append(key)
-
- for key in keys_order:
- in_data = attributes_to_set[key]
- attr = in_data["object"]
-
- # initial item definition
- item = {
- "name": key,
- "label": in_data["label"]
- }
-
- # cust attr type - may have different visualization
- type_name = attr["type"]["name"].lower()
- easy_types = ["text", "boolean", "date", "number"]
-
- easy_type = False
- if type_name in easy_types:
- easy_type = True
-
- elif type_name == "enumerator":
-
- attr_config = json.loads(attr["config"])
- attr_config_data = json.loads(attr_config["data"])
-
- if attr_config["multiSelect"] is True:
- multiselect_enumerators.append(self.item_splitter)
- multiselect_enumerators.append({
- "type": "label",
- "value": "{} ".format(in_data["label"])
- })
-
- default = in_data["default"]
- names = []
- for option in sorted(
- attr_config_data, key=lambda x: x["menu"]
- ):
- name = option["value"]
- new_name = "__{}__{}".format(key, name)
- names.append(new_name)
- item = {
- "name": new_name,
- "type": "boolean",
- "label": "- {}".format(option["menu"])
- }
- if default:
- if isinstance(default, (list, tuple)):
- if name in default:
- item["value"] = True
- else:
- if name == default:
- item["value"] = True
-
- multiselect_enumerators.append(item)
-
- multiselect_enumerators.append({
- "type": "hidden",
- "name": "__hidden__{}".format(key),
- "value": json.dumps(names)
- })
- else:
- easy_type = True
- item["data"] = attr_config_data
-
- else:
- self.log.warning((
- "Custom attribute \"{}\" has type \"{}\"."
- " I don't know how to handle"
- ).format(key, type_name))
- items.append({
- "type": "label",
- "value": (
- "!!! Can't handle Custom attritubte type \"{}\""
- " (key: \"{}\")"
- ).format(type_name, key)
- })
-
- if easy_type:
- item["type"] = type_name
-
- # default value in interface
- default = in_data["default"]
- if default is not None:
- item["value"] = default
-
- items.append(item)
-
- return items, multiselect_enumerators
-
- def launch(self, session, entities, event):
- in_data = event["data"].get("values")
- if not in_data:
- return
-
- create_project_structure_checked = in_data.pop(
- self.create_project_structure_key
- )
-
- root_values = {}
- root_key = "__root__"
- for key in tuple(in_data.keys()):
- if key.startswith(root_key):
- _key = key[len(root_key):]
- root_values[_key] = in_data.pop(key)
-
- root_names = in_data.pop("__rootnames__", None)
- root_data = {}
- for root_name in json.loads(root_names):
- root_data[root_name] = {}
- for key, value in tuple(root_values.items()):
- prefix = "{}__".format(root_name)
- if not key.startswith(prefix):
- continue
-
- _key = key[len(prefix):]
- root_data[root_name][_key] = value
-
- # Find hidden items for multiselect enumerators
- keys_to_process = []
- for key in in_data:
- if key.startswith("__hidden__"):
- keys_to_process.append(key)
-
- self.log.debug("Preparing data for Multiselect Enumerators")
- enumerators = {}
- for key in keys_to_process:
- new_key = key.replace("__hidden__", "")
- enumerator_items = in_data.pop(key)
- enumerators[new_key] = json.loads(enumerator_items)
-
- # find values set for multiselect enumerator
- for key, enumerator_items in enumerators.items():
- in_data[key] = []
-
- name = "__{}__".format(key)
-
- for item in enumerator_items:
- value = in_data.pop(item)
- if value is True:
- new_key = item.replace(name, "")
- in_data[key].append(new_key)
-
- self.log.debug("Setting Custom Attribute values")
-
- project_entity = entities[0]
- project_name = project_entity["full_name"]
-
- # Try to find project document
- dbcon = AvalonMongoDB()
- dbcon.install()
- dbcon.Session["AVALON_PROJECT"] = project_name
- project_doc = dbcon.find_one({
- "type": "project"
- })
- # Create project if is not available
- # - creation is required to be able set project anatomy and attributes
- if not project_doc:
- project_code = project_entity["name"]
- self.log.info("Creating project \"{} [{}]\"".format(
- project_name, project_code
- ))
- create_project(project_name, project_code, dbcon=dbcon)
-
- dbcon.uninstall()
-
- project_settings = ProjectSettings(project_name)
- project_anatomy_settings = project_settings["project_anatomy"]
- project_anatomy_settings["roots"] = root_data
-
- custom_attribute_values = {}
- attributes_entity = project_anatomy_settings["attributes"]
- for key, value in in_data.items():
- if key not in attributes_entity:
- custom_attribute_values[key] = value
- else:
- attributes_entity[key] = value
-
- try:
- project_settings.save()
- except SaveWarningExc as exc:
- self.log.info("Few warnings happened during settings save:")
- for warning in exc.warnings:
- self.log.info(str(warning))
-
- # Change custom attributes on project
- if custom_attribute_values:
- for key, value in custom_attribute_values.items():
- project_entity["custom_attributes"][key] = value
- self.log.debug("- Key \"{}\" set to \"{}\"".format(key, value))
- session.commit()
-
- # Trigger create project structure action
- if create_project_structure_checked:
- trigger_identifier = "{}.{}".format(
- self.create_project_structure_identifier,
- self.process_identifier()
- )
- self.trigger_action(trigger_identifier, event)
- return True
-
-
-def register(session):
- '''Register plugin. Called when used as an plugin.'''
- PrepareProjectLocal(session).register()
diff --git a/openpype/modules/default_modules/ftrack/event_handlers_user/action_sync_to_avalon.py b/openpype/modules/default_modules/ftrack/event_handlers_user/action_sync_to_avalon.py
deleted file mode 100644
index cd2f371f384..00000000000
--- a/openpype/modules/default_modules/ftrack/event_handlers_user/action_sync_to_avalon.py
+++ /dev/null
@@ -1,217 +0,0 @@
-import time
-import sys
-import json
-import traceback
-
-from openpype_modules.ftrack.lib import BaseAction, statics_icon
-from openpype_modules.ftrack.lib.avalon_sync import SyncEntitiesFactory
-
-
-class SyncToAvalonLocal(BaseAction):
- """
- Synchronizing data action - from Ftrack to Avalon DB
-
- Stores all information about entity.
- - Name(string) - Most important information = identifier of entity
- - Parent(ObjectId) - Avalon Project Id, if entity is not project itself
- - Data(dictionary):
- - VisualParent(ObjectId) - Avalon Id of parent asset
- - Parents(array of string) - All parent names except project
- - Tasks(array of string) - Tasks on asset
- - FtrackId(string)
- - entityType(string) - entity's type on Ftrack
- * All Custom attributes in group 'Avalon'
- - custom attributes that start with 'avalon_' are skipped
-
- * These information are stored for entities in whole project.
-
- Avalon ID of asset is stored to Ftrack
- - Custom attribute 'avalon_mongo_id'.
- - action IS NOT creating this Custom attribute if doesn't exist
- - run 'Create Custom Attributes' action
- - or do it manually (Not recommended)
- """
-
- identifier = "sync.to.avalon.local"
- label = "OpenPype Admin"
- variant = "- Sync To Avalon (Local)"
- priority = 200
- icon = statics_icon("ftrack", "action_icons", "OpenPypeAdmin.svg")
-
- settings_key = "sync_to_avalon_local"
-
- def __init__(self, *args, **kwargs):
- super().__init__(*args, **kwargs)
- self.entities_factory = SyncEntitiesFactory(self.log, self.session)
-
- def discover(self, session, entities, event):
- """ Validate selection. """
- is_valid = False
- for ent in event["data"]["selection"]:
- # Ignore entities that are not tasks or projects
- if ent["entityType"].lower() in ["show", "task"]:
- is_valid = True
- break
-
- if is_valid:
- is_valid = self.valid_roles(session, entities, event)
- return is_valid
-
- def launch(self, session, in_entities, event):
- self.log.debug("{}: Creating job".format(self.label))
-
- user_entity = session.query(
- "User where id is {}".format(event["source"]["user"]["id"])
- ).one()
- job_entity = session.create("Job", {
- "user": user_entity,
- "status": "running",
- "data": json.dumps({
- "description": "Sync to avalon is running..."
- })
- })
- session.commit()
-
- project_entity = self.get_project_from_entity(in_entities[0])
- project_name = project_entity["full_name"]
-
- try:
- result = self.synchronization(event, project_name)
-
- except Exception:
- self.log.error(
- "Synchronization failed due to code error", exc_info=True
- )
-
- description = "Sync to avalon Crashed (Download traceback)"
- self.add_traceback_to_job(
- job_entity, session, sys.exc_info(), description
- )
-
- msg = "An error has happened during synchronization"
- title = "Synchronization report ({}):".format(project_name)
- items = []
- items.append({
- "type": "label",
- "value": "# {}".format(msg)
- })
- items.append({
- "type": "label",
- "value": (
- "Download report from job for more information.
"
- )
- })
-
- report = {}
- try:
- report = self.entities_factory.report()
- except Exception:
- pass
-
- _items = report.get("items") or []
- if _items:
- items.append(self.entities_factory.report_splitter)
- items.extend(_items)
-
- self.show_interface(items, title, event, submit_btn_label="Ok")
-
- return {"success": True, "message": msg}
-
- job_entity["status"] = "done"
- job_entity["data"] = json.dumps({
- "description": "Sync to avalon finished."
- })
- session.commit()
-
- return result
-
- def synchronization(self, event, project_name):
- time_start = time.time()
-
- self.show_message(event, "Synchronization - Preparing data", True)
-
- try:
- output = self.entities_factory.launch_setup(project_name)
- if output is not None:
- return output
-
- time_1 = time.time()
-
- self.entities_factory.set_cutom_attributes()
- time_2 = time.time()
-
- # This must happen before all filtering!!!
- self.entities_factory.prepare_avalon_entities(project_name)
- time_3 = time.time()
-
- self.entities_factory.filter_by_ignore_sync()
- time_4 = time.time()
-
- self.entities_factory.duplicity_regex_check()
- time_5 = time.time()
-
- self.entities_factory.prepare_ftrack_ent_data()
- time_6 = time.time()
-
- self.entities_factory.synchronize()
- time_7 = time.time()
-
- self.log.debug(
- "*** Synchronization finished ***"
- )
- self.log.debug(
- "preparation <{}>".format(time_1 - time_start)
- )
- self.log.debug(
- "set_cutom_attributes <{}>".format(time_2 - time_1)
- )
- self.log.debug(
- "prepare_avalon_entities <{}>".format(time_3 - time_2)
- )
- self.log.debug(
- "filter_by_ignore_sync <{}>".format(time_4 - time_3)
- )
- self.log.debug(
- "duplicity_regex_check <{}>".format(time_5 - time_4)
- )
- self.log.debug(
- "prepare_ftrack_ent_data <{}>".format(time_6 - time_5)
- )
- self.log.debug(
- "synchronize <{}>".format(time_7 - time_6)
- )
- self.log.debug(
- "* Total time: {}".format(time_7 - time_start)
- )
-
- report = self.entities_factory.report()
- if report and report.get("items"):
- default_title = "Synchronization report ({}):".format(
- project_name
- )
- self.show_interface(
- items=report["items"],
- title=report.get("title", default_title),
- event=event
- )
- return {
- "success": True,
- "message": "Synchronization Finished"
- }
-
- finally:
- try:
- self.entities_factory.dbcon.uninstall()
- except Exception:
- pass
-
- try:
- self.entities_factory.session.close()
- except Exception:
- pass
-
-
-def register(session):
- '''Register plugin. Called when used as an plugin.'''
-
- SyncToAvalonLocal(session).register()
diff --git a/openpype/modules/default_modules/ftrack/ftrack_server/__init__.py b/openpype/modules/default_modules/ftrack/ftrack_server/__init__.py
deleted file mode 100644
index 9e3920b5003..00000000000
--- a/openpype/modules/default_modules/ftrack/ftrack_server/__init__.py
+++ /dev/null
@@ -1,8 +0,0 @@
-from .ftrack_server import FtrackServer
-from .lib import check_ftrack_url
-
-
-__all__ = (
- "FtrackServer",
- "check_ftrack_url"
-)
diff --git a/openpype/modules/default_modules/ftrack/ftrack_server/lib.py b/openpype/modules/default_modules/ftrack/ftrack_server/lib.py
deleted file mode 100644
index e80d6a3a6b1..00000000000
--- a/openpype/modules/default_modules/ftrack/ftrack_server/lib.py
+++ /dev/null
@@ -1,412 +0,0 @@
-import os
-import sys
-import logging
-import getpass
-import atexit
-import threading
-import datetime
-import time
-import queue
-import appdirs
-import pymongo
-
-import requests
-import ftrack_api
-import ftrack_api.session
-import ftrack_api.cache
-import ftrack_api.operation
-import ftrack_api._centralized_storage_scenario
-import ftrack_api.event
-from ftrack_api.logging import LazyLogMessage as L
-try:
- from weakref import WeakMethod
-except ImportError:
- from ftrack_api._weakref import WeakMethod
-from openpype_modules.ftrack.lib import get_ftrack_event_mongo_info
-
-from openpype.lib import OpenPypeMongoConnection
-from openpype.api import Logger
-
-TOPIC_STATUS_SERVER = "openpype.event.server.status"
-TOPIC_STATUS_SERVER_RESULT = "openpype.event.server.status.result"
-
-
-def check_ftrack_url(url, log_errors=True):
- """Checks if Ftrack server is responding"""
- if not url:
- print('ERROR: Ftrack URL is not set!')
- return None
-
- url = url.strip('/ ')
-
- if 'http' not in url:
- if url.endswith('ftrackapp.com'):
- url = 'https://' + url
- else:
- url = 'https://{0}.ftrackapp.com'.format(url)
- try:
- result = requests.get(url, allow_redirects=False)
- except requests.exceptions.RequestException:
- if log_errors:
- print('ERROR: Entered Ftrack URL is not accesible!')
- return False
-
- if (result.status_code != 200 or 'FTRACK_VERSION' not in result.headers):
- if log_errors:
- print('ERROR: Entered Ftrack URL is not accesible!')
- return False
-
- print('DEBUG: Ftrack server {} is accessible.'.format(url))
-
- return url
-
-
-class SocketBaseEventHub(ftrack_api.event.hub.EventHub):
-
- hearbeat_msg = b"hearbeat"
- heartbeat_callbacks = []
-
- def __init__(self, *args, **kwargs):
- self.sock = kwargs.pop("sock")
- super(SocketBaseEventHub, self).__init__(*args, **kwargs)
-
- def _handle_packet(self, code, packet_identifier, path, data):
- """Override `_handle_packet` which extend heartbeat"""
- code_name = self._code_name_mapping[code]
- if code_name == "heartbeat":
- # Reply with heartbeat.
- for callback in self.heartbeat_callbacks:
- callback()
-
- self.sock.sendall(self.hearbeat_msg)
- return self._send_packet(self._code_name_mapping["heartbeat"])
-
- return super(SocketBaseEventHub, self)._handle_packet(
- code, packet_identifier, path, data
- )
-
-
-class StatusEventHub(SocketBaseEventHub):
- def _handle_packet(self, code, packet_identifier, path, data):
- """Override `_handle_packet` which extend heartbeat"""
- code_name = self._code_name_mapping[code]
- if code_name == "connect":
- event = ftrack_api.event.base.Event(
- topic="openpype.status.started",
- data={},
- source={
- "id": self.id,
- "user": {"username": self._api_user}
- }
- )
- self._event_queue.put(event)
-
- return super(StatusEventHub, self)._handle_packet(
- code, packet_identifier, path, data
- )
-
-
-class StorerEventHub(SocketBaseEventHub):
-
- hearbeat_msg = b"storer"
-
- def _handle_packet(self, code, packet_identifier, path, data):
- """Override `_handle_packet` which extend heartbeat"""
- code_name = self._code_name_mapping[code]
- if code_name == "connect":
- event = ftrack_api.event.base.Event(
- topic="openpype.storer.started",
- data={},
- source={
- "id": self.id,
- "user": {"username": self._api_user}
- }
- )
- self._event_queue.put(event)
-
- return super(StorerEventHub, self)._handle_packet(
- code, packet_identifier, path, data
- )
-
-
-class ProcessEventHub(SocketBaseEventHub):
- hearbeat_msg = b"processor"
-
- is_collection_created = False
- pypelog = Logger().get_logger("Session Processor")
-
- def __init__(self, *args, **kwargs):
- self.mongo_url = None
- self.dbcon = None
-
- super(ProcessEventHub, self).__init__(*args, **kwargs)
-
- def prepare_dbcon(self):
- try:
- database_name, collection_name = get_ftrack_event_mongo_info()
- mongo_client = OpenPypeMongoConnection.get_mongo_client()
- self.dbcon = mongo_client[database_name][collection_name]
- self.mongo_client = mongo_client
-
- except pymongo.errors.AutoReconnect:
- self.pypelog.error((
- "Mongo server \"{}\" is not responding, exiting."
- ).format(OpenPypeMongoConnection.get_default_mongo_url()))
- sys.exit(0)
-
- except pymongo.errors.OperationFailure:
- self.pypelog.error((
- "Error with Mongo access, probably permissions."
- "Check if exist database with name \"{}\""
- " and collection \"{}\" inside."
- ).format(self.database, self.collection_name))
- self.sock.sendall(b"MongoError")
- sys.exit(0)
-
- def wait(self, duration=None):
- """Overriden wait
- Event are loaded from Mongo DB when queue is empty. Handled event is
- set as processed in Mongo DB.
- """
- started = time.time()
- self.prepare_dbcon()
- while True:
- try:
- event = self._event_queue.get(timeout=0.1)
- except queue.Empty:
- if not self.load_events():
- time.sleep(0.5)
- else:
- try:
- self._handle(event)
-
- mongo_id = event["data"].get("_event_mongo_id")
- if mongo_id is None:
- continue
-
- self.dbcon.update_one(
- {"_id": mongo_id},
- {"$set": {"pype_data.is_processed": True}}
- )
-
- except pymongo.errors.AutoReconnect:
- self.pypelog.error((
- "Mongo server \"{}\" is not responding, exiting."
- ).format(os.environ["AVALON_MONGO"]))
- sys.exit(0)
- # Additional special processing of events.
- if event['topic'] == 'ftrack.meta.disconnected':
- break
-
- if duration is not None:
- if (time.time() - started) > duration:
- break
-
- def load_events(self):
- """Load not processed events sorted by stored date"""
- ago_date = datetime.datetime.now() - datetime.timedelta(days=3)
- self.dbcon.delete_many({
- "pype_data.stored": {"$lte": ago_date},
- "pype_data.is_processed": True
- })
-
- not_processed_events = self.dbcon.find(
- {"pype_data.is_processed": False}
- ).sort(
- [("pype_data.stored", pymongo.ASCENDING)]
- )
-
- found = False
- for event_data in not_processed_events:
- new_event_data = {
- k: v for k, v in event_data.items()
- if k not in ["_id", "pype_data"]
- }
- try:
- event = ftrack_api.event.base.Event(**new_event_data)
- event["data"]["_event_mongo_id"] = event_data["_id"]
- except Exception:
- self.logger.exception(L(
- 'Failed to convert payload into event: {0}',
- event_data
- ))
- continue
- found = True
- self._event_queue.put(event)
-
- return found
-
- def _handle_packet(self, code, packet_identifier, path, data):
- """Override `_handle_packet` which skip events and extend heartbeat"""
- code_name = self._code_name_mapping[code]
- if code_name == "event":
- return
-
- return super()._handle_packet(code, packet_identifier, path, data)
-
-
-class CustomEventHubSession(ftrack_api.session.Session):
- '''An isolated session for interaction with an ftrack server.'''
- def __init__(
- self, server_url=None, api_key=None, api_user=None, auto_populate=True,
- plugin_paths=None, cache=None, cache_key_maker=None,
- auto_connect_event_hub=False, schema_cache_path=None,
- plugin_arguments=None, timeout=60, **kwargs
- ):
- self.kwargs = kwargs
-
- super(ftrack_api.session.Session, self).__init__()
- self.logger = logging.getLogger(
- __name__ + '.' + self.__class__.__name__
- )
- self._closed = False
-
- if server_url is None:
- server_url = os.environ.get('FTRACK_SERVER')
-
- if not server_url:
- raise TypeError(
- 'Required "server_url" not specified. Pass as argument or set '
- 'in environment variable FTRACK_SERVER.'
- )
-
- self._server_url = server_url
-
- if api_key is None:
- api_key = os.environ.get(
- 'FTRACK_API_KEY',
- # Backwards compatibility
- os.environ.get('FTRACK_APIKEY')
- )
-
- if not api_key:
- raise TypeError(
- 'Required "api_key" not specified. Pass as argument or set in '
- 'environment variable FTRACK_API_KEY.'
- )
-
- self._api_key = api_key
-
- if api_user is None:
- api_user = os.environ.get('FTRACK_API_USER')
- if not api_user:
- try:
- api_user = getpass.getuser()
- except Exception:
- pass
-
- if not api_user:
- raise TypeError(
- 'Required "api_user" not specified. Pass as argument, set in '
- 'environment variable FTRACK_API_USER or one of the standard '
- 'environment variables used by Python\'s getpass module.'
- )
-
- self._api_user = api_user
-
- # Currently pending operations.
- self.recorded_operations = ftrack_api.operation.Operations()
- self.record_operations = True
-
- self.cache_key_maker = cache_key_maker
- if self.cache_key_maker is None:
- self.cache_key_maker = ftrack_api.cache.StringKeyMaker()
-
- # Enforce always having a memory cache at top level so that the same
- # in-memory instance is returned from session.
- self.cache = ftrack_api.cache.LayeredCache([
- ftrack_api.cache.MemoryCache()
- ])
-
- if cache is not None:
- if callable(cache):
- cache = cache(self)
-
- if cache is not None:
- self.cache.caches.append(cache)
-
- self._managed_request = None
- self._request = requests.Session()
- self._request.auth = ftrack_api.session.SessionAuthentication(
- self._api_key, self._api_user
- )
- self.request_timeout = timeout
-
- self.auto_populate = auto_populate
-
- # Fetch server information and in doing so also check credentials.
- self._server_information = self._fetch_server_information()
-
- # Now check compatibility of server based on retrieved information.
- self.check_server_compatibility()
-
- # Construct event hub and load plugins.
- self._event_hub = self._create_event_hub()
-
- self._auto_connect_event_hub_thread = None
- if auto_connect_event_hub:
- # Connect to event hub in background thread so as not to block main
- # session usage waiting for event hub connection.
- self._auto_connect_event_hub_thread = threading.Thread(
- target=self._event_hub.connect
- )
- self._auto_connect_event_hub_thread.daemon = True
- self._auto_connect_event_hub_thread.start()
-
- # Register to auto-close session on exit.
- atexit.register(WeakMethod(self.close))
-
- self._plugin_paths = plugin_paths
- if self._plugin_paths is None:
- self._plugin_paths = os.environ.get(
- 'FTRACK_EVENT_PLUGIN_PATH', ''
- ).split(os.pathsep)
-
- self._discover_plugins(plugin_arguments=plugin_arguments)
-
- # TODO: Make schemas read-only and non-mutable (or at least without
- # rebuilding types)?
- if schema_cache_path is not False:
- if schema_cache_path is None:
- schema_cache_path = appdirs.user_cache_dir()
- schema_cache_path = os.environ.get(
- 'FTRACK_API_SCHEMA_CACHE_PATH', schema_cache_path
- )
-
- schema_cache_path = os.path.join(
- schema_cache_path, 'ftrack_api_schema_cache.json'
- )
-
- self.schemas = self._load_schemas(schema_cache_path)
- self.types = self._build_entity_type_classes(self.schemas)
-
- ftrack_api._centralized_storage_scenario.register(self)
-
- self._configure_locations()
- self.event_hub.publish(
- ftrack_api.event.base.Event(
- topic='ftrack.api.session.ready',
- data=dict(
- session=self
- )
- ),
- synchronous=True
- )
-
- def _create_event_hub(self):
- return ftrack_api.event.hub.EventHub(
- self._server_url,
- self._api_user,
- self._api_key
- )
-
-
-class SocketSession(CustomEventHubSession):
- def _create_event_hub(self):
- self.sock = self.kwargs["sock"]
- return self.kwargs["Eventhub"](
- self._server_url,
- self._api_user,
- self._api_key,
- sock=self.sock
- )
diff --git a/openpype/modules/default_modules/ftrack/launch_hooks/pre_python2_vendor.py b/openpype/modules/default_modules/ftrack/launch_hooks/pre_python2_vendor.py
deleted file mode 100644
index 0dd894bebf6..00000000000
--- a/openpype/modules/default_modules/ftrack/launch_hooks/pre_python2_vendor.py
+++ /dev/null
@@ -1,43 +0,0 @@
-import os
-from openpype.lib import PreLaunchHook
-from openpype_modules.ftrack import FTRACK_MODULE_DIR
-
-
-class PrePython2Support(PreLaunchHook):
- """Add python ftrack api module for Python 2 to PYTHONPATH.
-
- Path to vendor modules is added to the beggining of PYTHONPATH.
- """
-
- def execute(self):
- if not self.application.use_python_2:
- return
-
- self.log.info("Adding Ftrack Python 2 packages to PYTHONPATH.")
-
- # Prepare vendor dir path
- python_2_vendor = os.path.join(FTRACK_MODULE_DIR, "python2_vendor")
-
- # Add Python 2 modules
- python_paths = [
- # `python-ftrack-api`
- os.path.join(python_2_vendor, "ftrack-python-api", "source"),
- # `arrow`
- os.path.join(python_2_vendor, "arrow"),
- # `builtins` from `python-future`
- # - `python-future` is strict Python 2 module that cause crashes
- # of Python 3 scripts executed through OpenPype (burnin script etc.)
- os.path.join(python_2_vendor, "builtins"),
- # `backports.functools_lru_cache`
- os.path.join(
- python_2_vendor, "backports.functools_lru_cache"
- )
- ]
-
- # Load PYTHONPATH from current launch context
- python_path = self.launch_context.env.get("PYTHONPATH")
- if python_path:
- python_paths.append(python_path)
-
- # Set new PYTHONPATH to launch context environments
- self.launch_context.env["PYTHONPATH"] = os.pathsep.join(python_paths)
diff --git a/openpype/modules/default_modules/ftrack/lib/__init__.py b/openpype/modules/default_modules/ftrack/lib/__init__.py
deleted file mode 100644
index 80b4db9dd60..00000000000
--- a/openpype/modules/default_modules/ftrack/lib/__init__.py
+++ /dev/null
@@ -1,53 +0,0 @@
-from .constants import (
- CUST_ATTR_ID_KEY,
- CUST_ATTR_AUTO_SYNC,
- CUST_ATTR_GROUP,
- CUST_ATTR_TOOLS,
- CUST_ATTR_APPLICATIONS,
- CUST_ATTR_INTENT
-)
-from .settings import (
- get_ftrack_event_mongo_info
-)
-from .custom_attributes import (
- default_custom_attributes_definition,
- app_definitions_from_app_manager,
- tool_definitions_from_app_manager,
- get_openpype_attr,
- query_custom_attributes
-)
-
-from . import avalon_sync
-from . import credentials
-from .ftrack_base_handler import BaseHandler
-from .ftrack_event_handler import BaseEvent
-from .ftrack_action_handler import BaseAction, ServerAction, statics_icon
-
-
-__all__ = (
- "CUST_ATTR_ID_KEY",
- "CUST_ATTR_AUTO_SYNC",
- "CUST_ATTR_GROUP",
- "CUST_ATTR_TOOLS",
- "CUST_ATTR_APPLICATIONS",
-
- "get_ftrack_event_mongo_info",
-
- "default_custom_attributes_definition",
- "app_definitions_from_app_manager",
- "tool_definitions_from_app_manager",
- "get_openpype_attr",
- "query_custom_attributes",
-
- "avalon_sync",
-
- "credentials",
-
- "BaseHandler",
-
- "BaseEvent",
-
- "BaseAction",
- "ServerAction",
- "statics_icon"
-)
diff --git a/openpype/modules/default_modules/ftrack/lib/constants.py b/openpype/modules/default_modules/ftrack/lib/constants.py
deleted file mode 100644
index e6e2013d2bc..00000000000
--- a/openpype/modules/default_modules/ftrack/lib/constants.py
+++ /dev/null
@@ -1,14 +0,0 @@
-# Group name of custom attributes
-CUST_ATTR_GROUP = "openpype"
-
-# name of Custom attribute that stores mongo_id from avalon db
-CUST_ATTR_ID_KEY = "avalon_mongo_id"
-# Auto sync of project
-CUST_ATTR_AUTO_SYNC = "avalon_auto_sync"
-
-# Applications custom attribute name
-CUST_ATTR_APPLICATIONS = "applications"
-# Environment tools custom attribute
-CUST_ATTR_TOOLS = "tools_env"
-# Intent custom attribute name
-CUST_ATTR_INTENT = "intent"
diff --git a/openpype/modules/default_modules/ftrack/lib/credentials.py b/openpype/modules/default_modules/ftrack/lib/credentials.py
deleted file mode 100644
index 4e29e663829..00000000000
--- a/openpype/modules/default_modules/ftrack/lib/credentials.py
+++ /dev/null
@@ -1,105 +0,0 @@
-import os
-import ftrack_api
-
-try:
- from urllib.parse import urlparse
-except ImportError:
- from urlparse import urlparse
-
-
-from openpype.lib import OpenPypeSecureRegistry
-
-USERNAME_KEY = "username"
-API_KEY_KEY = "api_key"
-
-
-def get_ftrack_hostname(ftrack_server=None):
- if not ftrack_server:
- ftrack_server = os.environ.get("FTRACK_SERVER")
-
- if not ftrack_server:
- return None
-
- if "//" not in ftrack_server:
- ftrack_server = "//" + ftrack_server
-
- return urlparse(ftrack_server).hostname
-
-
-def _get_ftrack_secure_key(hostname, key):
- """Secure item key for entered hostname."""
- return "/".join(("ftrack", hostname, key))
-
-
-def get_credentials(ftrack_server=None):
- output = {
- USERNAME_KEY: None,
- API_KEY_KEY: None
- }
- hostname = get_ftrack_hostname(ftrack_server)
- if not hostname:
- return output
-
- username_name = _get_ftrack_secure_key(hostname, USERNAME_KEY)
- api_key_name = _get_ftrack_secure_key(hostname, API_KEY_KEY)
-
- username_registry = OpenPypeSecureRegistry(username_name)
- api_key_registry = OpenPypeSecureRegistry(api_key_name)
-
- output[USERNAME_KEY] = username_registry.get_item(USERNAME_KEY, None)
- output[API_KEY_KEY] = api_key_registry.get_item(API_KEY_KEY, None)
-
- return output
-
-
-def save_credentials(username, api_key, ftrack_server=None):
- hostname = get_ftrack_hostname(ftrack_server)
- username_name = _get_ftrack_secure_key(hostname, USERNAME_KEY)
- api_key_name = _get_ftrack_secure_key(hostname, API_KEY_KEY)
-
- # Clear credentials
- clear_credentials(ftrack_server)
-
- username_registry = OpenPypeSecureRegistry(username_name)
- api_key_registry = OpenPypeSecureRegistry(api_key_name)
-
- username_registry.set_item(USERNAME_KEY, username)
- api_key_registry.set_item(API_KEY_KEY, api_key)
-
-
-def clear_credentials(ftrack_server=None):
- hostname = get_ftrack_hostname(ftrack_server)
- username_name = _get_ftrack_secure_key(hostname, USERNAME_KEY)
- api_key_name = _get_ftrack_secure_key(hostname, API_KEY_KEY)
-
- username_registry = OpenPypeSecureRegistry(username_name)
- api_key_registry = OpenPypeSecureRegistry(api_key_name)
-
- current_username = username_registry.get_item(USERNAME_KEY, None)
- current_api_key = api_key_registry.get_item(API_KEY_KEY, None)
-
- if current_username is not None:
- username_registry.delete_item(USERNAME_KEY)
-
- if current_api_key is not None:
- api_key_registry.delete_item(API_KEY_KEY)
-
-
-def check_credentials(username, api_key, ftrack_server=None):
- if not ftrack_server:
- ftrack_server = os.environ.get("FTRACK_SERVER")
-
- if not ftrack_server or not username or not api_key:
- return False
-
- try:
- session = ftrack_api.Session(
- server_url=ftrack_server,
- api_key=api_key,
- api_user=username
- )
- session.close()
-
- except Exception:
- return False
- return True
diff --git a/openpype/modules/default_modules/ftrack/plugins/publish/collect_ftrack_api.py b/openpype/modules/default_modules/ftrack/plugins/publish/collect_ftrack_api.py
deleted file mode 100644
index a348617cfcd..00000000000
--- a/openpype/modules/default_modules/ftrack/plugins/publish/collect_ftrack_api.py
+++ /dev/null
@@ -1,210 +0,0 @@
-import os
-import logging
-import pyblish.api
-import avalon.api
-
-
-class CollectFtrackApi(pyblish.api.ContextPlugin):
- """ Collects an ftrack session and the current task id. """
-
- order = pyblish.api.CollectorOrder + 0.4999
- label = "Collect Ftrack Api"
-
- def process(self, context):
- ftrack_log = logging.getLogger('ftrack_api')
- ftrack_log.setLevel(logging.WARNING)
- ftrack_log = logging.getLogger('ftrack_api_old')
- ftrack_log.setLevel(logging.WARNING)
-
- # Collect session
- # NOTE Import python module here to know if import was successful
- import ftrack_api
-
- session = ftrack_api.Session(auto_connect_event_hub=False)
- self.log.debug("Ftrack user: \"{0}\"".format(session.api_user))
-
- # Collect task
- project_name = avalon.api.Session["AVALON_PROJECT"]
- asset_name = avalon.api.Session["AVALON_ASSET"]
- task_name = avalon.api.Session["AVALON_TASK"]
-
- # Find project entity
- project_query = 'Project where full_name is "{0}"'.format(project_name)
- self.log.debug("Project query: < {0} >".format(project_query))
- project_entities = list(session.query(project_query).all())
- if len(project_entities) == 0:
- raise AssertionError(
- "Project \"{0}\" not found in Ftrack.".format(project_name)
- )
- # QUESTION Is possible to happen?
- elif len(project_entities) > 1:
- raise AssertionError((
- "Found more than one project with name \"{0}\" in Ftrack."
- ).format(project_name))
-
- project_entity = project_entities[0]
- self.log.debug("Project found: {0}".format(project_entity))
-
- # Find asset entity
- entity_query = (
- 'TypedContext where project_id is "{0}"'
- ' and name is "{1}"'
- ).format(project_entity["id"], asset_name)
- self.log.debug("Asset entity query: < {0} >".format(entity_query))
- asset_entities = []
- for entity in session.query(entity_query).all():
- # Skip tasks
- if entity.entity_type.lower() != "task":
- asset_entities.append(entity)
-
- if len(asset_entities) == 0:
- raise AssertionError((
- "Entity with name \"{0}\" not found"
- " in Ftrack project \"{1}\"."
- ).format(asset_name, project_name))
-
- elif len(asset_entities) > 1:
- raise AssertionError((
- "Found more than one entity with name \"{0}\""
- " in Ftrack project \"{1}\"."
- ).format(asset_name, project_name))
-
- asset_entity = asset_entities[0]
- self.log.debug("Asset found: {0}".format(asset_entity))
-
- # Find task entity if task is set
- if task_name:
- task_query = (
- 'Task where name is "{0}" and parent_id is "{1}"'
- ).format(task_name, asset_entity["id"])
- self.log.debug("Task entity query: < {0} >".format(task_query))
- task_entity = session.query(task_query).first()
- if not task_entity:
- self.log.warning(
- "Task entity with name \"{0}\" was not found.".format(
- task_name
- )
- )
- else:
- self.log.debug("Task entity found: {0}".format(task_entity))
-
- else:
- task_entity = None
- self.log.warning("Task name is not set.")
-
- context.data["ftrackSession"] = session
- context.data["ftrackPythonModule"] = ftrack_api
- context.data["ftrackProject"] = project_entity
- context.data["ftrackEntity"] = asset_entity
- context.data["ftrackTask"] = task_entity
-
- self.per_instance_process(context, asset_name, task_name)
-
- def per_instance_process(
- self, context, context_asset_name, context_task_name
- ):
- instance_by_asset_and_task = {}
- for instance in context:
- self.log.debug(
- "Checking entities of instance \"{}\"".format(str(instance))
- )
- instance_asset_name = instance.data.get("asset")
- instance_task_name = instance.data.get("task")
-
- if not instance_asset_name and not instance_task_name:
- self.log.debug("Instance does not have set context keys.")
- continue
-
- elif instance_asset_name and instance_task_name:
- if (
- instance_asset_name == context_asset_name
- and instance_task_name == context_task_name
- ):
- self.log.debug((
- "Instance's context is same as in publish context."
- " Asset: {} | Task: {}"
- ).format(context_asset_name, context_task_name))
- continue
- asset_name = instance_asset_name
- task_name = instance_task_name
-
- elif instance_task_name:
- if instance_task_name == context_task_name:
- self.log.debug((
- "Instance's context task is same as in publish"
- " context. Task: {}"
- ).format(context_task_name))
- continue
-
- asset_name = context_asset_name
- task_name = instance_task_name
-
- elif instance_asset_name:
- if instance_asset_name == context_asset_name:
- self.log.debug((
- "Instance's context asset is same as in publish"
- " context. Asset: {}"
- ).format(context_asset_name))
- continue
-
- # Do not use context's task name
- task_name = instance_task_name
- asset_name = instance_asset_name
-
- if asset_name not in instance_by_asset_and_task:
- instance_by_asset_and_task[asset_name] = {}
-
- if task_name not in instance_by_asset_and_task[asset_name]:
- instance_by_asset_and_task[asset_name][task_name] = []
- instance_by_asset_and_task[asset_name][task_name].append(instance)
-
- if not instance_by_asset_and_task:
- return
-
- session = context.data["ftrackSession"]
- project_entity = context.data["ftrackProject"]
- asset_names = set()
- for asset_name in instance_by_asset_and_task.keys():
- asset_names.add(asset_name)
-
- joined_asset_names = ",".join([
- "\"{}\"".format(name)
- for name in asset_names
- ])
- entities = session.query((
- "TypedContext where project_id is \"{}\" and name in ({})"
- ).format(project_entity["id"], joined_asset_names)).all()
-
- entities_by_name = {
- entity["name"]: entity
- for entity in entities
- }
-
- for asset_name, by_task_data in instance_by_asset_and_task.items():
- entity = entities_by_name.get(asset_name)
- task_entity_by_name = {}
- if not entity:
- self.log.warning((
- "Didn't find entity with name \"{}\" in Project \"{}\""
- ).format(asset_name, project_entity["full_name"]))
- else:
- task_entities = session.query((
- "select id, name from Task where parent_id is \"{}\""
- ).format(entity["id"])).all()
- for task_entity in task_entities:
- task_name_low = task_entity["name"].lower()
- task_entity_by_name[task_name_low] = task_entity
-
- for task_name, instances in by_task_data.items():
- task_entity = None
- if task_name and entity:
- task_entity = task_entity_by_name.get(task_name.lower())
-
- for instance in instances:
- instance.data["ftrackEntity"] = entity
- instance.data["ftrackTask"] = task_entity
-
- self.log.debug((
- "Instance {} has own ftrack entities"
- " as has different context. TypedContext: {} Task: {}"
- ).format(str(instance), str(entity), str(task_entity)))
diff --git a/openpype/modules/default_modules/ftrack/plugins/publish/collect_username.py b/openpype/modules/default_modules/ftrack/plugins/publish/collect_username.py
deleted file mode 100644
index 7ea1c1f323c..00000000000
--- a/openpype/modules/default_modules/ftrack/plugins/publish/collect_username.py
+++ /dev/null
@@ -1,52 +0,0 @@
-"""Loads publishing context from json and continues in publish process.
-
-Requires:
- anatomy -> context["anatomy"] *(pyblish.api.CollectorOrder - 0.11)
-
-Provides:
- context, instances -> All data from previous publishing process.
-"""
-
-import ftrack_api
-import os
-
-import pyblish.api
-
-
-class CollectUsername(pyblish.api.ContextPlugin):
- """
- Translates user email to Ftrack username.
-
- Emails in Ftrack are same as company's Slack, username is needed to
- load data to Ftrack.
-
- Expects "pype.club" user created on Ftrack and FTRACK_BOT_API_KEY env
- var set up.
-
- """
- order = pyblish.api.CollectorOrder - 0.488
- label = "Collect ftrack username"
- hosts = ["webpublisher", "photoshop"]
- targets = ["remotepublish", "filespublish", "tvpaint_worker"]
-
- _context = None
-
- def process(self, context):
- self.log.info("CollectUsername")
-
- os.environ["FTRACK_API_USER"] = os.environ["FTRACK_BOT_API_USER"]
- os.environ["FTRACK_API_KEY"] = os.environ["FTRACK_BOT_API_KEY"]
-
- for instance in context:
- email = instance.data["user_email"]
- self.log.info("email:: {}".format(email))
- session = ftrack_api.Session(auto_connect_event_hub=False)
- user = session.query("User where email like '{}'".format(
- email))
-
- if not user:
- raise ValueError(
- "Couldnt find user with {} email".format(email))
-
- os.environ["FTRACK_API_USER"] = user[0].get("username")
- break
diff --git a/openpype/modules/default_modules/ftrack/plugins/publish/integrate_ftrack_api.py b/openpype/modules/default_modules/ftrack/plugins/publish/integrate_ftrack_api.py
deleted file mode 100644
index 6c25b9191e4..00000000000
--- a/openpype/modules/default_modules/ftrack/plugins/publish/integrate_ftrack_api.py
+++ /dev/null
@@ -1,466 +0,0 @@
-import os
-import sys
-import six
-import pyblish.api
-import clique
-
-
-class IntegrateFtrackApi(pyblish.api.InstancePlugin):
- """ Commit components to server. """
-
- order = pyblish.api.IntegratorOrder+0.499
- label = "Integrate Ftrack Api"
- families = ["ftrack"]
-
- def query(self, entitytype, data):
- """ Generate a query expression from data supplied.
-
- If a value is not a string, we'll add the id of the entity to the
- query.
-
- Args:
- entitytype (str): The type of entity to query.
- data (dict): The data to identify the entity.
- exclusions (list): All keys to exclude from the query.
-
- Returns:
- str: String query to use with "session.query"
- """
- queries = []
- if sys.version_info[0] < 3:
- for key, value in data.iteritems():
- if not isinstance(value, (basestring, int)):
- self.log.info("value: {}".format(value))
- if "id" in value.keys():
- queries.append(
- "{0}.id is \"{1}\"".format(key, value["id"])
- )
- else:
- queries.append("{0} is \"{1}\"".format(key, value))
- else:
- for key, value in data.items():
- if not isinstance(value, (str, int)):
- self.log.info("value: {}".format(value))
- if "id" in value.keys():
- queries.append(
- "{0}.id is \"{1}\"".format(key, value["id"])
- )
- else:
- queries.append("{0} is \"{1}\"".format(key, value))
-
- query = (
- "select id from " + entitytype + " where " + " and ".join(queries)
- )
- self.log.debug(query)
- return query
-
- def _set_task_status(self, instance, task_entity, session):
- project_entity = instance.context.data.get("ftrackProject")
- if not project_entity:
- self.log.info("Task status won't be set, project is not known.")
- return
-
- if not task_entity:
- self.log.info("Task status won't be set, task is not known.")
- return
-
- status_name = instance.context.data.get("ftrackStatus")
- if not status_name:
- self.log.info("Ftrack status name is not set.")
- return
-
- self.log.debug(
- "Ftrack status name will be (maybe) set to \"{}\"".format(
- status_name
- )
- )
-
- project_schema = project_entity["project_schema"]
- task_statuses = project_schema.get_statuses(
- "Task", task_entity["type_id"]
- )
- task_statuses_by_low_name = {
- status["name"].lower(): status for status in task_statuses
- }
- status = task_statuses_by_low_name.get(status_name.lower())
- if not status:
- self.log.warning((
- "Task status \"{}\" won't be set,"
- " status is now allowed on task type \"{}\"."
- ).format(status_name, task_entity["type"]["name"]))
- return
-
- self.log.info("Setting task status to \"{}\"".format(status_name))
- task_entity["status"] = status
- try:
- session.commit()
- except Exception:
- tp, value, tb = sys.exc_info()
- session.rollback()
- session._configure_locations()
- six.reraise(tp, value, tb)
-
- def process(self, instance):
- session = instance.context.data["ftrackSession"]
- context = instance.context
-
- name = None
- # If instance has set "ftrackEntity" or "ftrackTask" then use them from
- # instance. Even if they are set to None. If they are set to None it
- # has a reason. (like has different context)
- if "ftrackEntity" in instance.data or "ftrackTask" in instance.data:
- task = instance.data.get("ftrackTask")
- parent = instance.data.get("ftrackEntity")
-
- elif "ftrackEntity" in context.data or "ftrackTask" in context.data:
- task = context.data.get("ftrackTask")
- parent = context.data.get("ftrackEntity")
-
- if task:
- parent = task["parent"]
- name = task
- elif parent:
- name = parent["name"]
-
- if not name:
- self.log.info((
- "Skipping ftrack integration. Instance \"{}\" does not"
- " have specified ftrack entities."
- ).format(str(instance)))
- return
-
- info_msg = (
- "Created new {entity_type} with data: {data}"
- ", metadata: {metadata}."
- )
-
- used_asset_versions = []
-
- self._set_task_status(instance, task, session)
-
- # Iterate over components and publish
- for data in instance.data.get("ftrackComponentsList", []):
- # AssetType
- # Get existing entity.
- assettype_data = {"short": "upload"}
- assettype_data.update(data.get("assettype_data", {}))
- self.log.debug("data: {}".format(data))
-
- assettype_entity = session.query(
- self.query("AssetType", assettype_data)
- ).first()
-
- # Create a new entity if none exits.
- if not assettype_entity:
- assettype_entity = session.create("AssetType", assettype_data)
- self.log.debug("Created new AssetType with data: {}".format(
- assettype_data
- ))
-
- # Asset
- # Get existing entity.
- asset_data = {
- "name": name,
- "type": assettype_entity,
- "parent": parent,
- }
- asset_data.update(data.get("asset_data", {}))
-
- asset_entity = session.query(
- self.query("Asset", asset_data)
- ).first()
-
- self.log.info("asset entity: {}".format(asset_entity))
-
- # Extracting metadata, and adding after entity creation. This is
- # due to a ftrack_api bug where you can't add metadata on creation.
- asset_metadata = asset_data.pop("metadata", {})
-
- # Create a new entity if none exits.
- if not asset_entity:
- asset_entity = session.create("Asset", asset_data)
- self.log.debug(
- info_msg.format(
- entity_type="Asset",
- data=asset_data,
- metadata=asset_metadata
- )
- )
- try:
- session.commit()
- except Exception:
- tp, value, tb = sys.exc_info()
- session.rollback()
- session._configure_locations()
- six.reraise(tp, value, tb)
-
- # Adding metadata
- existing_asset_metadata = asset_entity["metadata"]
- existing_asset_metadata.update(asset_metadata)
- asset_entity["metadata"] = existing_asset_metadata
-
- # AssetVersion
- # Get existing entity.
- assetversion_data = {
- "version": 0,
- "asset": asset_entity,
- }
- _assetversion_data = data.get("assetversion_data", {})
- assetversion_cust_attrs = _assetversion_data.pop(
- "custom_attributes", {}
- )
- asset_version_comment = _assetversion_data.pop(
- "comment", None
- )
- assetversion_data.update(_assetversion_data)
-
- assetversion_entity = session.query(
- self.query("AssetVersion", assetversion_data)
- ).first()
-
- # Extracting metadata, and adding after entity creation. This is
- # due to a ftrack_api bug where you can't add metadata on creation.
- assetversion_metadata = assetversion_data.pop("metadata", {})
-
- if task:
- assetversion_data['task'] = task
-
- # Create a new entity if none exits.
- if not assetversion_entity:
- assetversion_entity = session.create(
- "AssetVersion", assetversion_data
- )
- self.log.debug(
- info_msg.format(
- entity_type="AssetVersion",
- data=assetversion_data,
- metadata=assetversion_metadata
- )
- )
- try:
- session.commit()
- except Exception:
- tp, value, tb = sys.exc_info()
- session.rollback()
- session._configure_locations()
- six.reraise(tp, value, tb)
-
- # Adding metadata
- existing_assetversion_metadata = assetversion_entity["metadata"]
- existing_assetversion_metadata.update(assetversion_metadata)
- assetversion_entity["metadata"] = existing_assetversion_metadata
-
- # Add comment
- if asset_version_comment:
- assetversion_entity["comment"] = asset_version_comment
- try:
- session.commit()
- except Exception:
- session.rollback()
- session._configure_locations()
- self.log.warning((
- "Comment was not possible to set for AssetVersion"
- "\"{0}\". Can't set it's value to: \"{1}\""
- ).format(
- assetversion_entity["id"], str(asset_version_comment)
- ))
-
- # Adding Custom Attributes
- for attr, val in assetversion_cust_attrs.items():
- if attr in assetversion_entity["custom_attributes"]:
- try:
- assetversion_entity["custom_attributes"][attr] = val
- session.commit()
- continue
- except Exception:
- session.rollback()
- session._configure_locations()
-
- self.log.warning((
- "Custom Attrubute \"{0}\""
- " is not available for AssetVersion <{1}>."
- " Can't set it's value to: \"{2}\""
- ).format(attr, assetversion_entity["id"], str(val)))
-
- # Have to commit the version and asset, because location can't
- # determine the final location without.
- try:
- session.commit()
- except Exception:
- tp, value, tb = sys.exc_info()
- session.rollback()
- session._configure_locations()
- six.reraise(tp, value, tb)
-
- # Component
- # Get existing entity.
- component_data = {
- "name": "main",
- "version": assetversion_entity
- }
- component_data.update(data.get("component_data", {}))
-
- component_entity = session.query(
- self.query("Component", component_data)
- ).first()
-
- component_overwrite = data.get("component_overwrite", False)
- location = data.get("component_location", session.pick_location())
-
- # Overwrite existing component data if requested.
- if component_entity and component_overwrite:
-
- origin_location = session.query(
- "Location where name is \"ftrack.origin\""
- ).one()
-
- # Removing existing members from location
- components = list(component_entity.get("members", []))
- components += [component_entity]
- for component in components:
- for loc in component["component_locations"]:
- if location["id"] == loc["location_id"]:
- location.remove_component(
- component, recursive=False
- )
-
- # Deleting existing members on component entity
- for member in component_entity.get("members", []):
- session.delete(member)
- del(member)
-
- try:
- session.commit()
- except Exception:
- tp, value, tb = sys.exc_info()
- session.rollback()
- session._configure_locations()
- six.reraise(tp, value, tb)
-
- # Reset members in memory
- if "members" in component_entity.keys():
- component_entity["members"] = []
-
- # Add components to origin location
- try:
- collection = clique.parse(data["component_path"])
- except ValueError:
- # Assume its a single file
- # Changing file type
- name, ext = os.path.splitext(data["component_path"])
- component_entity["file_type"] = ext
-
- origin_location.add_component(
- component_entity, data["component_path"]
- )
- else:
- # Changing file type
- component_entity["file_type"] = collection.format("{tail}")
-
- # Create member components for sequence.
- for member_path in collection:
-
- size = 0
- try:
- size = os.path.getsize(member_path)
- except OSError:
- pass
-
- name = collection.match(member_path).group("index")
-
- member_data = {
- "name": name,
- "container": component_entity,
- "size": size,
- "file_type": os.path.splitext(member_path)[-1]
- }
-
- component = session.create(
- "FileComponent", member_data
- )
- origin_location.add_component(
- component, member_path, recursive=False
- )
- component_entity["members"].append(component)
-
- # Add components to location.
- location.add_component(
- component_entity, origin_location, recursive=True
- )
-
- data["component"] = component_entity
- msg = "Overwriting Component with path: {0}, data: {1}, "
- msg += "location: {2}"
- self.log.info(
- msg.format(
- data["component_path"],
- component_data,
- location
- )
- )
-
- # Extracting metadata, and adding after entity creation. This is
- # due to a ftrack_api bug where you can't add metadata on creation.
- component_metadata = component_data.pop("metadata", {})
-
- # Create new component if none exists.
- new_component = False
- if not component_entity:
- component_entity = assetversion_entity.create_component(
- data["component_path"],
- data=component_data,
- location=location
- )
- data["component"] = component_entity
- msg = "Created new Component with path: {0}, data: {1}"
- msg += ", metadata: {2}, location: {3}"
- self.log.info(
- msg.format(
- data["component_path"],
- component_data,
- component_metadata,
- location
- )
- )
- new_component = True
-
- # Adding metadata
- existing_component_metadata = component_entity["metadata"]
- existing_component_metadata.update(component_metadata)
- component_entity["metadata"] = existing_component_metadata
-
- # if component_data['name'] = 'ftrackreview-mp4-mp4':
- # assetversion_entity["thumbnail_id"]
-
- # Setting assetversion thumbnail
- if data.get("thumbnail", False):
- assetversion_entity["thumbnail_id"] = component_entity["id"]
-
- # Inform user about no changes to the database.
- if (component_entity and not component_overwrite and
- not new_component):
- data["component"] = component_entity
- self.log.info(
- "Found existing component, and no request to overwrite. "
- "Nothing has been changed."
- )
- else:
- # Commit changes.
- try:
- session.commit()
- except Exception:
- tp, value, tb = sys.exc_info()
- session.rollback()
- session._configure_locations()
- six.reraise(tp, value, tb)
-
- if assetversion_entity not in used_asset_versions:
- used_asset_versions.append(assetversion_entity)
-
- asset_versions_key = "ftrackIntegratedAssetVersions"
- if asset_versions_key not in instance.data:
- instance.data[asset_versions_key] = []
-
- for asset_version in used_asset_versions:
- if asset_version not in instance.data[asset_versions_key]:
- instance.data[asset_versions_key].append(asset_version)
diff --git a/openpype/modules/default_modules/ftrack/plugins/publish/integrate_ftrack_instances.py b/openpype/modules/default_modules/ftrack/plugins/publish/integrate_ftrack_instances.py
deleted file mode 100644
index 8399e19184c..00000000000
--- a/openpype/modules/default_modules/ftrack/plugins/publish/integrate_ftrack_instances.py
+++ /dev/null
@@ -1,266 +0,0 @@
-import os
-import json
-import copy
-import pyblish.api
-
-
-class IntegrateFtrackInstance(pyblish.api.InstancePlugin):
- """Collect ftrack component data (not integrate yet).
-
- Add ftrack component list to instance.
- """
-
- order = pyblish.api.IntegratorOrder + 0.48
- label = "Integrate Ftrack Component"
- families = ["ftrack"]
-
- family_mapping = {
- "camera": "cam",
- "look": "look",
- "mayaascii": "scene",
- "model": "geo",
- "rig": "rig",
- "setdress": "setdress",
- "pointcache": "cache",
- "render": "render",
- "render2d": "render",
- "nukescript": "comp",
- "write": "render",
- "review": "mov",
- "plate": "img",
- "audio": "audio",
- "workfile": "scene",
- "animation": "cache",
- "image": "img",
- "reference": "reference"
- }
-
- def process(self, instance):
- self.log.debug("instance {}".format(instance))
-
- instance_version = instance.data.get("version")
- if instance_version is None:
- raise ValueError("Instance version not set")
-
- version_number = int(instance_version)
-
- family = instance.data["family"]
- family_low = instance.data["family"].lower()
-
- asset_type = instance.data.get("ftrackFamily")
- if not asset_type and family_low in self.family_mapping:
- asset_type = self.family_mapping[family_low]
-
- # Ignore this instance if neither "ftrackFamily" or a family mapping is
- # found.
- if not asset_type:
- self.log.info((
- "Family \"{}\" does not match any asset type mapping"
- ).format(family))
- return
-
- instance_repres = instance.data.get("representations")
- if not instance_repres:
- self.log.info((
- "Skipping instance. Does not have any representations {}"
- ).format(str(instance)))
- return
-
- # Prepare FPS
- instance_fps = instance.data.get("fps")
- if instance_fps is None:
- instance_fps = instance.context.data["fps"]
-
- # Base of component item data
- # - create a copy of this object when want to use it
- base_component_item = {
- "assettype_data": {
- "short": asset_type,
- },
- "asset_data": {
- "name": instance.data["subset"],
- },
- "assetversion_data": {
- "version": version_number,
- "comment": instance.context.data.get("comment") or ""
- },
- "component_overwrite": False,
- # This can be change optionally
- "thumbnail": False,
- # These must be changed for each component
- "component_data": None,
- "component_path": None,
- "component_location": None
- }
-
- ft_session = instance.context.data["ftrackSession"]
-
- # Filter types of representations
- review_representations = []
- thumbnail_representations = []
- other_representations = []
- for repre in instance_repres:
- self.log.debug("Representation {}".format(repre))
- repre_tags = repre.get("tags") or []
- if repre.get("thumbnail") or "thumbnail" in repre_tags:
- thumbnail_representations.append(repre)
-
- elif "ftrackreview" in repre_tags:
- review_representations.append(repre)
-
- else:
- other_representations.append(repre)
-
- # Prepare ftrack locations
- unmanaged_location = ft_session.query(
- "Location where name is \"ftrack.unmanaged\""
- ).one()
- ftrack_server_location = ft_session.query(
- "Location where name is \"ftrack.server\""
- ).one()
-
- # Components data
- component_list = []
- # Components that will be duplicated to unmanaged location
- src_components_to_add = []
-
- # Create thumbnail components
- # TODO what if there is multiple thumbnails?
- first_thumbnail_component = None
- for repre in thumbnail_representations:
- published_path = repre.get("published_path")
- if not published_path:
- comp_files = repre["files"]
- if isinstance(comp_files, (tuple, list, set)):
- filename = comp_files[0]
- else:
- filename = comp_files
-
- published_path = os.path.join(
- repre["stagingDir"], filename
- )
- if not os.path.exists(published_path):
- continue
- repre["published_path"] = published_path
-
- # Create copy of base comp item and append it
- thumbnail_item = copy.deepcopy(base_component_item)
- thumbnail_item["component_path"] = repre["published_path"]
- thumbnail_item["component_data"] = {
- "name": "thumbnail"
- }
- thumbnail_item["thumbnail"] = True
- # Create copy of item before setting location
- src_components_to_add.append(copy.deepcopy(thumbnail_item))
- # Create copy of first thumbnail
- if first_thumbnail_component is None:
- first_thumbnail_component = copy.deepcopy(thumbnail_item)
- # Set location
- thumbnail_item["component_location"] = ftrack_server_location
- # Add item to component list
- component_list.append(thumbnail_item)
-
- # Create review components
- # Change asset name of each new component for review
- is_first_review_repre = True
- not_first_components = []
- for repre in review_representations:
- frame_start = repre.get("frameStartFtrack")
- frame_end = repre.get("frameEndFtrack")
- if frame_start is None or frame_end is None:
- frame_start = instance.data["frameStart"]
- frame_end = instance.data["frameEnd"]
-
- # Frame end of uploaded video file should be duration in frames
- # - frame start is always 0
- # - frame end is duration in frames
- duration = frame_end - frame_start + 1
-
- fps = repre.get("fps")
- if fps is None:
- fps = instance_fps
-
- # Create copy of base comp item and append it
- review_item = copy.deepcopy(base_component_item)
- # Change location
- review_item["component_path"] = repre["published_path"]
- # Change component data
- review_item["component_data"] = {
- # Default component name is "main".
- "name": "ftrackreview-mp4",
- "metadata": {
- "ftr_meta": json.dumps({
- "frameIn": 0,
- "frameOut": int(duration),
- "frameRate": float(fps)
- })
- }
- }
- # Create copy of item before setting location or changing asset
- src_components_to_add.append(copy.deepcopy(review_item))
- if is_first_review_repre:
- is_first_review_repre = False
- else:
- # Add representation name to asset name of "not first" review
- asset_name = review_item["asset_data"]["name"]
- review_item["asset_data"]["name"] = "_".join(
- (asset_name, repre["name"])
- )
- not_first_components.append(review_item)
-
- # Set location
- review_item["component_location"] = ftrack_server_location
- # Add item to component list
- component_list.append(review_item)
-
- # Duplicate thumbnail component for all not first reviews
- if first_thumbnail_component is not None:
- for component_item in not_first_components:
- asset_name = component_item["asset_data"]["name"]
- new_thumbnail_component = copy.deepcopy(
- first_thumbnail_component
- )
- new_thumbnail_component["asset_data"]["name"] = asset_name
- new_thumbnail_component["component_location"] = (
- ftrack_server_location
- )
- component_list.append(new_thumbnail_component)
-
- # Add source components for review and thubmnail components
- for copy_src_item in src_components_to_add:
- # Make sure thumbnail is disabled
- copy_src_item["thumbnail"] = False
- # Set location
- copy_src_item["component_location"] = unmanaged_location
- # Modify name of component to have suffix "_src"
- component_data = copy_src_item["component_data"]
- component_name = component_data["name"]
- component_data["name"] = component_name + "_src"
- component_list.append(copy_src_item)
-
- # Add others representations as component
- for repre in other_representations:
- published_path = repre.get("published_path")
- if not published_path:
- continue
- # Create copy of base comp item and append it
- other_item = copy.deepcopy(base_component_item)
- other_item["component_data"] = {
- "name": repre["name"]
- }
- other_item["component_location"] = unmanaged_location
- other_item["component_path"] = published_path
- component_list.append(other_item)
-
- def json_obj_parser(obj):
- return str(obj)
-
- self.log.debug("Components list: {}".format(
- json.dumps(
- component_list,
- sort_keys=True,
- indent=4,
- default=json_obj_parser
- )
- ))
- instance.data["ftrackComponentsList"] = component_list
diff --git a/openpype/modules/default_modules/ftrack/plugins/publish/integrate_ftrack_note.py b/openpype/modules/default_modules/ftrack/plugins/publish/integrate_ftrack_note.py
deleted file mode 100644
index acd295854da..00000000000
--- a/openpype/modules/default_modules/ftrack/plugins/publish/integrate_ftrack_note.py
+++ /dev/null
@@ -1,149 +0,0 @@
-import sys
-import json
-import pyblish.api
-import six
-
-
-class IntegrateFtrackNote(pyblish.api.InstancePlugin):
- """Create comments in Ftrack."""
-
- # Must be after integrate asset new
- order = pyblish.api.IntegratorOrder + 0.4999
- label = "Integrate Ftrack note"
- families = ["ftrack"]
- optional = True
-
- # Can be set in presets:
- # - Allows only `intent` and `comment` keys
- note_with_intent_template = "{intent}: {comment}"
- # - note label must exist in Ftrack
- note_labels = []
-
- def get_intent_label(self, session, intent_value):
- if not intent_value:
- return
-
- intent_configurations = session.query(
- "CustomAttributeConfiguration where key is intent"
- ).all()
- if not intent_configurations:
- return
-
- intent_configuration = intent_configurations[0]
- if len(intent_configuration) > 1:
- self.log.warning((
- "Found more than one `intent` custom attribute."
- " Using first found."
- ))
-
- config = intent_configuration.get("config")
- if not config:
- return
-
- configuration = json.loads(config)
- items = configuration.get("data")
- if not items:
- return
-
- if sys.version_info[0] < 3:
- string_type = basestring
- else:
- string_type = str
-
- if isinstance(items, string_type):
- items = json.loads(items)
-
- intent_label = None
- for item in items:
- if item["value"] == intent_value:
- intent_label = item["menu"]
- break
-
- return intent_label
-
- def process(self, instance):
- comment = (instance.context.data.get("comment") or "").strip()
- if not comment:
- self.log.info("Comment is not set.")
- return
-
- self.log.debug("Comment is set to `{}`".format(comment))
-
- session = instance.context.data["ftrackSession"]
-
- intent = instance.context.data.get("intent")
- if intent and isinstance(intent, dict):
- intent_val = intent.get("value")
- intent_label = intent.get("label")
- else:
- intent_val = intent_label = intent
-
- final_label = None
- if intent_val:
- final_label = self.get_intent_label(session, intent_val)
- if final_label is None:
- final_label = intent_label
-
- # if intent label is set then format comment
- # - it is possible that intent_label is equal to "" (empty string)
- if final_label:
- msg = "Intent label is set to `{}`.".format(final_label)
- comment = self.note_with_intent_template.format(**{
- "intent": final_label,
- "comment": comment
- })
-
- elif intent_val:
- msg = (
- "Intent is set to `{}` and was not added"
- " to comment because label is set to `{}`."
- ).format(intent_val, final_label)
-
- else:
- msg = "Intent is not set."
-
- self.log.debug(msg)
-
- asset_versions_key = "ftrackIntegratedAssetVersions"
- asset_versions = instance.data.get(asset_versions_key)
- if not asset_versions:
- self.log.info("There are any integrated AssetVersions")
- return
-
- user = session.query(
- "User where username is \"{}\"".format(session.api_user)
- ).first()
- if not user:
- self.log.warning(
- "Was not able to query current User {}".format(
- session.api_user
- )
- )
-
- labels = []
- if self.note_labels:
- all_labels = session.query("NoteLabel").all()
- labels_by_low_name = {lab["name"].lower(): lab for lab in all_labels}
- for _label in self.note_labels:
- label = labels_by_low_name.get(_label.lower())
- if not label:
- self.log.warning(
- "Note Label `{}` was not found.".format(_label)
- )
- continue
-
- labels.append(label)
-
- for asset_version in asset_versions:
- asset_version.create_note(comment, author=user, labels=labels)
-
- try:
- session.commit()
- self.log.debug("Note added to AssetVersion \"{}\"".format(
- str(asset_version)
- ))
- except Exception:
- tp, value, tb = sys.exc_info()
- session.rollback()
- session._configure_locations()
- six.reraise(tp, value, tb)
diff --git a/openpype/modules/default_modules/ftrack/plugins/publish/integrate_hierarchy_ftrack.py b/openpype/modules/default_modules/ftrack/plugins/publish/integrate_hierarchy_ftrack.py
deleted file mode 100644
index fbd64d9f703..00000000000
--- a/openpype/modules/default_modules/ftrack/plugins/publish/integrate_hierarchy_ftrack.py
+++ /dev/null
@@ -1,356 +0,0 @@
-import sys
-import collections
-import six
-import pyblish.api
-from avalon import io
-
-# Copy of constant `openpype_modules.ftrack.lib.avalon_sync.CUST_ATTR_AUTO_SYNC`
-CUST_ATTR_AUTO_SYNC = "avalon_auto_sync"
-CUST_ATTR_GROUP = "openpype"
-
-
-# Copy of `get_pype_attr` from openpype_modules.ftrack.lib
-# TODO import from openpype's ftrack module when possible to not break Python 2
-def get_pype_attr(session, split_hierarchical=True):
- custom_attributes = []
- hier_custom_attributes = []
- # TODO remove deprecated "avalon" group from query
- cust_attrs_query = (
- "select id, entity_type, object_type_id, is_hierarchical, default"
- " from CustomAttributeConfiguration"
- # Kept `pype` for Backwards Compatiblity
- " where group.name in (\"pype\", \"{}\")"
- ).format(CUST_ATTR_GROUP)
- all_avalon_attr = session.query(cust_attrs_query).all()
- for cust_attr in all_avalon_attr:
- if split_hierarchical and cust_attr["is_hierarchical"]:
- hier_custom_attributes.append(cust_attr)
- continue
-
- custom_attributes.append(cust_attr)
-
- if split_hierarchical:
- # return tuple
- return custom_attributes, hier_custom_attributes
-
- return custom_attributes
-
-
-class IntegrateHierarchyToFtrack(pyblish.api.ContextPlugin):
- """
- Create entities in ftrack based on collected data from premiere
- Example of entry data:
- {
- "ProjectXS": {
- "entity_type": "Project",
- "custom_attributes": {
- "fps": 24,...
- },
- "tasks": [
- "Compositing",
- "Lighting",... *task must exist as task type in project schema*
- ],
- "childs": {
- "sq01": {
- "entity_type": "Sequence",
- ...
- }
- }
- }
- }
- """
-
- order = pyblish.api.IntegratorOrder - 0.04
- label = 'Integrate Hierarchy To Ftrack'
- families = ["shot"]
- hosts = ["hiero", "resolve", "standalonepublisher"]
- optional = False
-
- def process(self, context):
- self.context = context
- if "hierarchyContext" not in self.context.data:
- return
-
- hierarchy_context = self.context.data["hierarchyContext"]
-
- self.session = self.context.data["ftrackSession"]
- project_name = self.context.data["projectEntity"]["name"]
- query = 'Project where full_name is "{}"'.format(project_name)
- project = self.session.query(query).one()
- auto_sync_state = project[
- "custom_attributes"][CUST_ATTR_AUTO_SYNC]
-
- if not io.Session:
- io.install()
-
- self.ft_project = None
-
- input_data = hierarchy_context
-
- # disable termporarily ftrack project's autosyncing
- if auto_sync_state:
- self.auto_sync_off(project)
-
- try:
- # import ftrack hierarchy
- self.import_to_ftrack(input_data)
- except Exception:
- raise
- finally:
- if auto_sync_state:
- self.auto_sync_on(project)
-
- def import_to_ftrack(self, input_data, parent=None):
- # Prequery hiearchical custom attributes
- hier_custom_attributes = get_pype_attr(self.session)[1]
- hier_attr_by_key = {
- attr["key"]: attr
- for attr in hier_custom_attributes
- }
- # Get ftrack api module (as they are different per python version)
- ftrack_api = self.context.data["ftrackPythonModule"]
-
- for entity_name in input_data:
- entity_data = input_data[entity_name]
- entity_type = entity_data['entity_type']
- self.log.debug(entity_data)
- self.log.debug(entity_type)
-
- if entity_type.lower() == 'project':
- query = 'Project where full_name is "{}"'.format(entity_name)
- entity = self.session.query(query).one()
- self.ft_project = entity
- self.task_types = self.get_all_task_types(entity)
-
- elif self.ft_project is None or parent is None:
- raise AssertionError(
- "Collected items are not in right order!"
- )
-
- # try to find if entity already exists
- else:
- query = (
- 'TypedContext where name is "{0}" and '
- 'project_id is "{1}"'
- ).format(entity_name, self.ft_project["id"])
- try:
- entity = self.session.query(query).one()
- except Exception:
- entity = None
-
- # Create entity if not exists
- if entity is None:
- entity = self.create_entity(
- name=entity_name,
- type=entity_type,
- parent=parent
- )
- # self.log.info('entity: {}'.format(dict(entity)))
- # CUSTOM ATTRIBUTES
- custom_attributes = entity_data.get('custom_attributes', [])
- instances = [
- i for i in self.context if i.data['asset'] in entity['name']
- ]
- for key in custom_attributes:
- hier_attr = hier_attr_by_key.get(key)
- # Use simple method if key is not hierarchical
- if not hier_attr:
- assert (key in entity['custom_attributes']), (
- 'Missing custom attribute key: `{0}` in attrs: '
- '`{1}`'.format(key, entity['custom_attributes'].keys())
- )
-
- entity['custom_attributes'][key] = custom_attributes[key]
-
- else:
- # Use ftrack operations method to set hiearchical
- # attribute value.
- # - this is because there may be non hiearchical custom
- # attributes with different properties
- entity_key = collections.OrderedDict()
- entity_key["configuration_id"] = hier_attr["id"]
- entity_key["entity_id"] = entity["id"]
- self.session.recorded_operations.push(
- ftrack_api.operation.UpdateEntityOperation(
- "ContextCustomAttributeValue",
- entity_key,
- "value",
- ftrack_api.symbol.NOT_SET,
- custom_attributes[key]
- )
- )
-
- for instance in instances:
- instance.data['ftrackEntity'] = entity
-
- try:
- self.session.commit()
- except Exception:
- tp, value, tb = sys.exc_info()
- self.session.rollback()
- self.session._configure_locations()
- six.reraise(tp, value, tb)
-
- # TASKS
- tasks = entity_data.get('tasks', [])
- existing_tasks = []
- tasks_to_create = []
- for child in entity['children']:
- if child.entity_type.lower() == 'task':
- existing_tasks.append(child['name'].lower())
- # existing_tasks.append(child['type']['name'])
-
- for task_name in tasks:
- task_type = tasks[task_name]["type"]
- if task_name.lower() in existing_tasks:
- print("Task {} already exists".format(task_name))
- continue
- tasks_to_create.append((task_name, task_type))
-
- for task_name, task_type in tasks_to_create:
- self.create_task(
- name=task_name,
- task_type=task_type,
- parent=entity
- )
- try:
- self.session.commit()
- except Exception:
- tp, value, tb = sys.exc_info()
- self.session.rollback()
- self.session._configure_locations()
- six.reraise(tp, value, tb)
-
- # Incoming links.
- self.create_links(entity_data, entity)
- try:
- self.session.commit()
- except Exception:
- tp, value, tb = sys.exc_info()
- self.session.rollback()
- self.session._configure_locations()
- six.reraise(tp, value, tb)
-
- # Create notes.
- user = self.session.query(
- "User where username is \"{}\"".format(self.session.api_user)
- ).first()
- if user:
- for comment in entity_data.get("comments", []):
- entity.create_note(comment, user)
- else:
- self.log.warning(
- "Was not able to query current User {}".format(
- self.session.api_user
- )
- )
- try:
- self.session.commit()
- except Exception:
- tp, value, tb = sys.exc_info()
- self.session.rollback()
- self.session._configure_locations()
- six.reraise(tp, value, tb)
-
- # Import children.
- if 'childs' in entity_data:
- self.import_to_ftrack(
- entity_data['childs'], entity)
-
- def create_links(self, entity_data, entity):
- # Clear existing links.
- for link in entity.get("incoming_links", []):
- self.session.delete(link)
- try:
- self.session.commit()
- except Exception:
- tp, value, tb = sys.exc_info()
- self.session.rollback()
- self.session._configure_locations()
- six.reraise(tp, value, tb)
-
- # Create new links.
- for input in entity_data.get("inputs", []):
- input_id = io.find_one({"_id": input})["data"]["ftrackId"]
- assetbuild = self.session.get("AssetBuild", input_id)
- self.log.debug(
- "Creating link from {0} to {1}".format(
- assetbuild["name"], entity["name"]
- )
- )
- self.session.create(
- "TypedContextLink", {"from": assetbuild, "to": entity}
- )
-
- def get_all_task_types(self, project):
- tasks = {}
- proj_template = project['project_schema']
- temp_task_types = proj_template['_task_type_schema']['types']
-
- for type in temp_task_types:
- if type['name'] not in tasks:
- tasks[type['name']] = type
-
- return tasks
-
- def create_task(self, name, task_type, parent):
- task = self.session.create('Task', {
- 'name': name,
- 'parent': parent
- })
- # TODO not secured!!! - check if task_type exists
- self.log.info(task_type)
- self.log.info(self.task_types)
- task['type'] = self.task_types[task_type]
-
- try:
- self.session.commit()
- except Exception:
- tp, value, tb = sys.exc_info()
- self.session.rollback()
- self.session._configure_locations()
- six.reraise(tp, value, tb)
-
- return task
-
- def create_entity(self, name, type, parent):
- entity = self.session.create(type, {
- 'name': name,
- 'parent': parent
- })
- try:
- self.session.commit()
- except Exception:
- tp, value, tb = sys.exc_info()
- self.session.rollback()
- self.session._configure_locations()
- six.reraise(tp, value, tb)
-
- return entity
-
- def auto_sync_off(self, project):
- project["custom_attributes"][CUST_ATTR_AUTO_SYNC] = False
-
- self.log.info("Ftrack autosync swithed off")
-
- try:
- self.session.commit()
- except Exception:
- tp, value, tb = sys.exc_info()
- self.session.rollback()
- self.session._configure_locations()
- six.reraise(tp, value, tb)
-
- def auto_sync_on(self, project):
-
- project["custom_attributes"][CUST_ATTR_AUTO_SYNC] = True
-
- self.log.info("Ftrack autosync swithed on")
-
- try:
- self.session.commit()
- except Exception:
- tp, value, tb = sys.exc_info()
- self.session.rollback()
- self.session._configure_locations()
- six.reraise(tp, value, tb)
diff --git a/openpype/modules/default_modules/ftrack/python2_vendor/arrow b/openpype/modules/default_modules/ftrack/python2_vendor/arrow
deleted file mode 160000
index b746fedf728..00000000000
--- a/openpype/modules/default_modules/ftrack/python2_vendor/arrow
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit b746fedf7286c3755a46f07ab72f4c414cd41fc0
diff --git a/openpype/modules/default_modules/ftrack/python2_vendor/ftrack-python-api b/openpype/modules/default_modules/ftrack/python2_vendor/ftrack-python-api
deleted file mode 160000
index d277f474ab0..00000000000
--- a/openpype/modules/default_modules/ftrack/python2_vendor/ftrack-python-api
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit d277f474ab016e7b53479c36af87cb861d0cc53e
diff --git a/openpype/modules/default_modules/royal_render/api.py b/openpype/modules/default_modules/royal_render/api.py
deleted file mode 100644
index ed9e71f2404..00000000000
--- a/openpype/modules/default_modules/royal_render/api.py
+++ /dev/null
@@ -1,199 +0,0 @@
-# -*- coding: utf-8 -*-
-"""Wrapper around Royal Render API."""
-import sys
-import os
-
-from openpype.settings import get_project_settings
-from openpype.lib.local_settings import OpenPypeSettingsRegistry
-from openpype.lib import PypeLogger, run_subprocess
-from .rr_job import RRJob, SubmitFile, SubmitterParameter
-
-
-log = PypeLogger.get_logger("RoyalRender")
-
-
-class Api:
-
- _settings = None
- RR_SUBMIT_CONSOLE = 1
- RR_SUBMIT_API = 2
-
- def __init__(self, settings, project=None):
- self._settings = settings
- self._initialize_rr(project)
-
- def _initialize_rr(self, project=None):
- # type: (str) -> None
- """Initialize RR Path.
-
- Args:
- project (str, Optional): Project name to set RR api in
- context.
-
- """
- if project:
- project_settings = get_project_settings(project)
- rr_path = (
- project_settings
- ["royalrender"]
- ["rr_paths"]
- )
- else:
- rr_path = (
- self._settings
- ["modules"]
- ["royalrender"]
- ["rr_path"]
- ["default"]
- )
- os.environ["RR_ROOT"] = rr_path
- self._rr_path = rr_path
-
- def _get_rr_bin_path(self, rr_root=None):
- # type: (str) -> str
- """Get path to RR bin folder."""
- rr_root = rr_root or self._rr_path
- is_64bit_python = sys.maxsize > 2 ** 32
-
- rr_bin_path = ""
- if sys.platform.lower() == "win32":
- rr_bin_path = "/bin/win64"
- if not is_64bit_python:
- # we are using 64bit python
- rr_bin_path = "/bin/win"
- rr_bin_path = rr_bin_path.replace(
- "/", os.path.sep
- )
-
- if sys.platform.lower() == "darwin":
- rr_bin_path = "/bin/mac64"
- if not is_64bit_python:
- rr_bin_path = "/bin/mac"
-
- if sys.platform.lower() == "linux":
- rr_bin_path = "/bin/lx64"
-
- return os.path.join(rr_root, rr_bin_path)
-
- def _initialize_module_path(self):
- # type: () -> None
- """Set RR modules for Python."""
- # default for linux
- rr_bin = self._get_rr_bin_path()
- rr_module_path = os.path.join(rr_bin, "lx64/lib")
-
- if sys.platform.lower() == "win32":
- rr_module_path = rr_bin
- rr_module_path = rr_module_path.replace(
- "/", os.path.sep
- )
-
- if sys.platform.lower() == "darwin":
- rr_module_path = os.path.join(rr_bin, "lib/python/27")
-
- sys.path.append(os.path.join(self._rr_path, rr_module_path))
-
- def create_submission(self, jobs, submitter_attributes, file_name=None):
- # type: (list[RRJob], list[SubmitterParameter], str) -> SubmitFile
- """Create jobs submission file.
-
- Args:
- jobs (list): List of :class:`RRJob`
- submitter_attributes (list): List of submitter attributes
- :class:`SubmitterParameter` for whole submission batch.
- file_name (str), optional): File path to write data to.
-
- Returns:
- str: XML data of job submission files.
-
- """
- raise NotImplementedError
-
- def submit_file(self, file, mode=RR_SUBMIT_CONSOLE):
- # type: (SubmitFile, int) -> None
- if mode == self.RR_SUBMIT_CONSOLE:
- self._submit_using_console(file)
-
- # RR v7 supports only Python 2.7 so we bail out in fear
- # until there is support for Python 3 😰
- raise NotImplementedError(
- "Submission via RoyalRender API is not supported yet")
- # self._submit_using_api(file)
-
- def _submit_using_console(self, file):
- # type: (SubmitFile) -> bool
- rr_console = os.path.join(
- self._get_rr_bin_path(),
- "rrSubmitterconsole"
- )
-
- if sys.platform.lower() == "darwin":
- if "/bin/mac64" in rr_console:
- rr_console = rr_console.replace("/bin/mac64", "/bin/mac")
-
- if sys.platform.lower() == "win32":
- if "/bin/win64" in rr_console:
- rr_console = rr_console.replace("/bin/win64", "/bin/win")
- rr_console += ".exe"
-
- args = [rr_console, file]
- run_subprocess(" ".join(args), logger=log)
-
- def _submit_using_api(self, file):
- # type: (SubmitFile) -> None
- """Use RR API to submit jobs.
-
- Args:
- file (SubmitFile): Submit jobs definition.
-
- Throws:
- RoyalRenderException: When something fails.
-
- """
- self._initialize_module_path()
- import libpyRR2 as rrLib # noqa
- from rrJob import getClass_JobBasics # noqa
- import libpyRR2 as _RenderAppBasic # noqa
-
- tcp = rrLib._rrTCP("") # noqa
- rr_server = tcp.getRRServer()
-
- if len(rr_server) == 0:
- log.info("Got RR IP address {}".format(rr_server))
-
- # TODO: Port is hardcoded in RR? If not, move it to Settings
- if not tcp.setServer(rr_server, 7773):
- log.error(
- "Can not set RR server: {}".format(tcp.errorMessage()))
- raise RoyalRenderException(tcp.errorMessage())
-
- # TODO: This need UI and better handling of username/password.
- # We can't store password in keychain as it is pulled multiple
- # times and users on linux must enter keychain password every time.
- # Probably best way until we setup our own user management would be
- # to encrypt password and save it to json locally. Not bulletproof
- # but at least it is not stored in plaintext.
- reg = OpenPypeSettingsRegistry()
- try:
- rr_user = reg.get_item("rr_username")
- rr_password = reg.get_item("rr_password")
- except ValueError:
- # user has no rr credentials set
- pass
- else:
- # login to RR
- tcp.setLogin(rr_user, rr_password)
-
- job = getClass_JobBasics()
- renderer = _RenderAppBasic()
-
- # iterate over SubmitFile, set _JobBasic (job) and renderer
- # and feed it to jobSubmitNew()
- # not implemented yet
- job.renderer = renderer
- tcp.jobSubmitNew(job)
-
-
-class RoyalRenderException(Exception):
- """Exception used in various error states coming from RR."""
- pass
diff --git a/openpype/modules/default_modules/sync_server/sync_server_module.py b/openpype/modules/default_modules/sync_server/sync_server_module.py
deleted file mode 100644
index 500203f3fc3..00000000000
--- a/openpype/modules/default_modules/sync_server/sync_server_module.py
+++ /dev/null
@@ -1,1808 +0,0 @@
-import os
-from bson.objectid import ObjectId
-from datetime import datetime
-import threading
-import platform
-import copy
-from collections import deque
-
-from avalon.api import AvalonMongoDB
-
-from openpype.modules import OpenPypeModule
-from openpype_interfaces import ITrayModule
-from openpype.api import (
- Anatomy,
- get_project_settings,
- get_system_settings,
- get_local_site_id)
-from openpype.lib import PypeLogger
-from openpype.settings.lib import (
- get_default_anatomy_settings,
- get_anatomy_settings)
-
-from .providers.local_drive import LocalDriveHandler
-from .providers import lib
-
-from .utils import time_function, SyncStatus
-
-
-log = PypeLogger().get_logger("SyncServer")
-
-
-class SyncServerModule(OpenPypeModule, ITrayModule):
- """
- Synchronization server that is syncing published files from local to
- any of implemented providers (like GDrive, S3 etc.)
- Runs in the background and checks all representations, looks for files
- that are marked to be in different location than 'studio' (temporary),
- checks if 'created_dt' field is present denoting successful sync
- with provider destination.
- Sites structure is created during publish OR by calling 'add_site'
- method.
-
- By default it will always contain 1 record with
- "name" == self.presets["active_site"] and
- filled "created_dt" AND 1 or multiple records for all defined
- remote sites, where "created_dt" is not present.
- This highlights that file should be uploaded to
- remote destination
-
- ''' - example of synced file test_Cylinder_lookMain_v010.ma to GDrive
- "files" : [
- {
- "path" : "{root}/Test/Assets/Cylinder/publish/look/lookMain/v010/
- test_Cylinder_lookMain_v010.ma",
- "_id" : ObjectId("5eeb25e411e06a16209ab78f"),
- "hash" : "test_Cylinder_lookMain_v010,ma|1592468963,24|4822",
- "size" : NumberLong(4822),
- "sites" : [
- {
- "name": "john_local_XD4345",
- "created_dt" : ISODate("2020-05-22T08:05:44.000Z")
- },
- {
- "id" : ObjectId("5eeb25e411e06a16209ab78f"),
- "name": "gdrive",
- "created_dt" : ISODate("2020-05-55T08:54:35.833Z")
- ]
- }
- },
- '''
- Each Tray app has assigned its own self.presets["local_id"]
- used in sites as a name.
- Tray is searching only for records where name matches its
- self.presets["active_site"] + self.presets["remote_site"].
- "active_site" could be storage in studio ('studio'), or specific
- "local_id" when user is working disconnected from home.
- If the local record has its "created_dt" filled, it is a source and
- process will try to upload the file to all defined remote sites.
-
- Remote files "id" is real id that could be used in appropriate API.
- Local files have "id" too, for conformity, contains just file name.
- It is expected that multiple providers will be implemented in separate
- classes and registered in 'providers.py'.
-
- """
- # limit querying DB to look for X number of representations that should
- # be sync, we try to run more loops with less records
- # actual number of files synced could be lower as providers can have
- # different limits imposed by its API
- # set 0 to no limit
- REPRESENTATION_LIMIT = 100
- DEFAULT_SITE = 'studio'
- LOCAL_SITE = 'local'
- LOG_PROGRESS_SEC = 5 # how often log progress to DB
- DEFAULT_PRIORITY = 50 # higher is better, allowed range 1 - 1000
-
- name = "sync_server"
- label = "Sync Queue"
-
- def initialize(self, module_settings):
- """
- Called during Module Manager creation.
-
- Collects needed data, checks asyncio presence.
- Sets 'enabled' according to global settings for the module.
- Shouldnt be doing any initialization, thats a job for 'tray_init'
- """
- self.enabled = module_settings[self.name]["enabled"]
-
- # some parts of code need to run sequentially, not in async
- self.lock = None
- self._sync_system_settings = None
- # settings for all enabled projects for sync
- self._sync_project_settings = None
- self.sync_server_thread = None # asyncio requires new thread
-
- self.action_show_widget = None
- self._paused = False
- self._paused_projects = set()
- self._paused_representations = set()
- self._anatomies = {}
-
- self._connection = None
-
- # list of long blocking tasks
- self.long_running_tasks = deque()
- # projects that long tasks are running on
- self.projects_processed = set()
-
- """ Start of Public API """
- def add_site(self, collection, representation_id, site_name=None,
- force=False):
- """
- Adds new site to representation to be synced.
-
- 'collection' must have synchronization enabled (globally or
- project only)
-
- Used as a API endpoint from outside applications (Loader etc)
-
- Args:
- collection (string): project name (must match DB)
- representation_id (string): MongoDB _id value
- site_name (string): name of configured and active site
- force (bool): reset site if exists
-
- Returns:
- throws ValueError if any issue
- """
- if not self.get_sync_project_setting(collection):
- raise ValueError("Project not configured")
-
- if not site_name:
- site_name = self.DEFAULT_SITE
-
- self.reset_site_on_representation(collection,
- representation_id,
- site_name=site_name, force=force)
-
- # public facing API
- def remove_site(self, collection, representation_id, site_name,
- remove_local_files=False):
- """
- Removes 'site_name' for particular 'representation_id' on
- 'collection'
-
- Args:
- collection (string): project name (must match DB)
- representation_id (string): MongoDB _id value
- site_name (string): name of configured and active site
- remove_local_files (bool): remove only files for 'local_id'
- site
-
- Returns:
- throws ValueError if any issue
- """
- if not self.get_sync_project_setting(collection):
- raise ValueError("Project not configured")
-
- self.reset_site_on_representation(collection,
- representation_id,
- site_name=site_name,
- remove=True)
- if remove_local_files:
- self._remove_local_file(collection, representation_id, site_name)
-
- def clear_project(self, collection, site_name):
- """
- Clear 'collection' of 'site_name' and its local files
-
- Works only on real local sites, not on 'studio'
- """
- query = {
- "type": "representation",
- "files.sites.name": site_name
- }
-
- representations = list(
- self.connection.database[collection].find(query))
- if not representations:
- self.log.debug("No repre found")
- return
-
- for repre in representations:
- self.remove_site(collection, repre.get("_id"), site_name, True)
-
- def create_validate_project_task(self, collection, site_name):
- """Adds metadata about project files validation on a queue.
-
- This process will loop through all representation and check if
- their files actually exist on an active site.
-
- This might be useful for edge cases when artists is switching
- between sites, remote site is actually physically mounted and
- active site has same file urls etc.
-
- Task will run on a asyncio loop, shouldn't be blocking.
- """
- task = {
- "type": "validate",
- "project_name": collection,
- "func": lambda: self.validate_project(collection, site_name)
- }
- self.projects_processed.add(collection)
- self.long_running_tasks.append(task)
-
- def validate_project(self, collection, site_name, remove_missing=False):
- """
- Validate 'collection' of 'site_name' and its local files
-
- If file present and not marked with a 'site_name' in DB, DB is
- updated with site name and file modified date.
-
- Args:
- module (SyncServerModule)
- collection (string): project name
- site_name (string): active site name
- remove_missing (bool): if True remove sites in DB if missing
- physically
- """
- self.log.debug("Validation of {} for {} started".format(collection,
- site_name))
- query = {
- "type": "representation"
- }
-
- representations = list(
- self.connection.database[collection].find(query))
- if not representations:
- self.log.debug("No repre found")
- return
-
- sites_added = 0
- sites_removed = 0
- for repre in representations:
- repre_id = repre["_id"]
- for repre_file in repre.get("files", []):
- try:
- has_site = site_name in [site["name"]
- for site in repre_file["sites"]]
- except TypeError:
- self.log.debug("Structure error in {}".format(repre_id))
- continue
-
- if has_site and not remove_missing:
- continue
-
- file_path = repre_file.get("path", "")
- local_file_path = self.get_local_file_path(collection,
- site_name,
- file_path)
-
- if local_file_path and os.path.exists(local_file_path):
- self.log.debug("Adding site {} for {}".format(site_name,
- repre_id))
- if not has_site:
- query = {
- "_id": repre_id
- }
- created_dt = datetime.fromtimestamp(
- os.path.getmtime(local_file_path))
- elem = {"name": site_name,
- "created_dt": created_dt}
- self._add_site(collection, query, [repre], elem,
- site_name=site_name,
- file_id=repre_file["_id"])
- sites_added += 1
- else:
- if has_site and remove_missing:
- self.log.debug("Removing site {} for {}".
- format(site_name, repre_id))
- self.reset_provider_for_file(collection,
- repre_id,
- file_id=repre_file["_id"],
- remove=True)
- sites_removed += 1
-
- if sites_added % 100 == 0:
- self.log.debug("Sites added {}".format(sites_added))
-
- self.log.debug("Validation of {} for {} ended".format(collection,
- site_name))
- self.log.info("Sites added {}, sites removed {}".format(sites_added,
- sites_removed))
-
- def pause_representation(self, collection, representation_id, site_name):
- """
- Sets 'representation_id' as paused, eg. no syncing should be
- happening on it.
-
- Args:
- collection (string): project name
- representation_id (string): MongoDB objectId value
- site_name (string): 'gdrive', 'studio' etc.
- """
- log.info("Pausing SyncServer for {}".format(representation_id))
- self._paused_representations.add(representation_id)
- self.reset_site_on_representation(collection, representation_id,
- site_name=site_name, pause=True)
-
- def unpause_representation(self, collection, representation_id, site_name):
- """
- Sets 'representation_id' as unpaused.
-
- Does not fail or warn if repre wasn't paused.
-
- Args:
- collection (string): project name
- representation_id (string): MongoDB objectId value
- site_name (string): 'gdrive', 'studio' etc.
- """
- log.info("Unpausing SyncServer for {}".format(representation_id))
- try:
- self._paused_representations.remove(representation_id)
- except KeyError:
- pass
- # self.paused_representations is not persistent
- self.reset_site_on_representation(collection, representation_id,
- site_name=site_name, pause=False)
-
- def is_representation_paused(self, representation_id,
- check_parents=False, project_name=None):
- """
- Returns if 'representation_id' is paused or not.
-
- Args:
- representation_id (string): MongoDB objectId value
- check_parents (bool): check if parent project or server itself
- are not paused
- project_name (string): project to check if paused
-
- if 'check_parents', 'project_name' should be set too
- Returns:
- (bool)
- """
- condition = representation_id in self._paused_representations
- if check_parents and project_name:
- condition = condition or \
- self.is_project_paused(project_name) or \
- self.is_paused()
- return condition
-
- def pause_project(self, project_name):
- """
- Sets 'project_name' as paused, eg. no syncing should be
- happening on all representation inside.
-
- Args:
- project_name (string): collection name
- """
- log.info("Pausing SyncServer for {}".format(project_name))
- self._paused_projects.add(project_name)
-
- def unpause_project(self, project_name):
- """
- Sets 'project_name' as unpaused
-
- Does not fail or warn if project wasn't paused.
-
- Args:
- project_name (string): collection name
- """
- log.info("Unpausing SyncServer for {}".format(project_name))
- try:
- self._paused_projects.remove(project_name)
- except KeyError:
- pass
-
- def is_project_paused(self, project_name, check_parents=False):
- """
- Returns if 'project_name' is paused or not.
-
- Args:
- project_name (string): collection name
- check_parents (bool): check if server itself
- is not paused
- Returns:
- (bool)
- """
- condition = project_name in self._paused_projects
- if check_parents:
- condition = condition or self.is_paused()
- return condition
-
- def pause_server(self):
- """
- Pause sync server
-
- It won't check anything, not uploading/downloading...
- """
- log.info("Pausing SyncServer")
- self._paused = True
-
- def unpause_server(self):
- """
- Unpause server
- """
- log.info("Unpausing SyncServer")
- self._paused = False
-
- def is_paused(self):
- """ Is server paused """
- return self._paused
-
- def get_active_sites(self, project_name):
- """
- Returns list of active sites for 'project_name'.
-
- By default it returns ['studio'], this site is default
- and always present even if SyncServer is not enabled. (for publish)
-
- Used mainly for Local settings for user override.
-
- Args:
- project_name (string):
-
- Returns:
- (list) of strings
- """
- return self.get_active_sites_from_settings(
- get_project_settings(project_name))
-
- def get_active_sites_from_settings(self, settings):
- """
- List available active sites from incoming 'settings'. Used for
- returning 'default' values for Local Settings
-
- Args:
- settings (dict): full settings (global + project)
- Returns:
- (list) of strings
- """
- sync_settings = self._parse_sync_settings_from_settings(settings)
-
- return self._get_enabled_sites_from_settings(sync_settings)
-
- def get_active_site(self, project_name):
- """
- Returns active (mine) site for 'project_name' from settings
-
- Returns:
- (string)
- """
- active_site = self.get_sync_project_setting(
- project_name)['config']['active_site']
- if active_site == self.LOCAL_SITE:
- return get_local_site_id()
- return active_site
-
- # remote sites
- def get_remote_sites(self, project_name):
- """
- Returns all remote sites configured on 'project_name'.
-
- If 'project_name' is not enabled for syncing returns [].
-
- Used by Local setting to allow user choose remote site.
-
- Args:
- project_name (string):
-
- Returns:
- (list) of strings
- """
- return self.get_remote_sites_from_settings(
- get_project_settings(project_name))
-
- def get_remote_sites_from_settings(self, settings):
- """
- Get remote sites for returning 'default' values for Local Settings
- """
- sync_settings = self._parse_sync_settings_from_settings(settings)
-
- return self._get_remote_sites_from_settings(sync_settings)
-
- def get_remote_site(self, project_name):
- """
- Returns remote (theirs) site for 'project_name' from settings
- """
- remote_site = self.get_sync_project_setting(
- project_name)['config']['remote_site']
- if remote_site == self.LOCAL_SITE:
- return get_local_site_id()
-
- return remote_site
-
- def get_local_normalized_site(self, site_name):
- """
- Return 'site_name' or 'local' if 'site_name' is local id.
-
- In some places Settings or Local Settings require 'local' instead
- of real site name.
- """
- if site_name == get_local_site_id():
- site_name = self.LOCAL_SITE
-
- return site_name
-
- # Methods for Settings UI to draw appropriate forms
- @classmethod
- def get_system_settings_schema(cls):
- """ Gets system level schema of configurable items
-
- Used for Setting UI to provide forms.
- """
- ret_dict = {}
- for provider_code in lib.factory.providers:
- ret_dict[provider_code] = \
- lib.factory.get_provider_cls(provider_code). \
- get_system_settings_schema()
-
- return ret_dict
-
- @classmethod
- def get_project_settings_schema(cls):
- """ Gets project level schema of configurable items.
-
- It is not using Setting! Used for Setting UI to provide forms.
- """
- ret_dict = {}
- for provider_code in lib.factory.providers:
- ret_dict[provider_code] = \
- lib.factory.get_provider_cls(provider_code). \
- get_project_settings_schema()
-
- return ret_dict
-
- @classmethod
- def get_local_settings_schema(cls):
- """ Gets local level schema of configurable items.
-
- It is not using Setting! Used for Setting UI to provide forms.
- """
- ret_dict = {}
- for provider_code in lib.factory.providers:
- ret_dict[provider_code] = \
- lib.factory.get_provider_cls(provider_code). \
- get_local_settings_schema()
-
- return ret_dict
-
- # Needs to be refactored after Settings are updated
- # # Methods for Settings to get appriate values to fill forms
- # def get_configurable_items(self, scope=None):
- # """
- # Returns list of sites that could be configurable for all projects
- #
- # Could be filtered by 'scope' argument (list)
- #
- # Args:
- # scope (list of utils.EditableScope)
- #
- # Returns:
- # (dict of list of dict)
- # {
- # siteA : [
- # {
- # key:"root", label:"root",
- # "value":"{'work': 'c:/projects'}",
- # "type": "dict",
- # "children":[
- # { "key": "work",
- # "type": "text",
- # "value": "c:/projects"}
- # ]
- # },
- # {
- # key:"credentials_url", label:"Credentials url",
- # "value":"'c:/projects/cred.json'", "type": "text", # noqa: E501
- # "namespace": "{project_setting}/global/sync_server/ # noqa: E501
- # sites"
- # }
- # ]
- # }
- # """
- # editable = {}
- # applicable_projects = list(self.connection.projects())
- # applicable_projects.append(None)
- # for project in applicable_projects:
- # project_name = None
- # if project:
- # project_name = project["name"]
- #
- # items = self.get_configurable_items_for_project(project_name,
- # scope)
- # editable.update(items)
- #
- # return editable
- #
- # def get_local_settings_schema_for_project(self, project_name):
- # """Wrapper for Local settings - for specific 'project_name'"""
- # return self.get_configurable_items_for_project(project_name,
- # EditableScopes.LOCAL)
- #
- # def get_configurable_items_for_project(self, project_name=None,
- # scope=None):
- # """
- # Returns list of items that could be configurable for specific
- # 'project_name'
- #
- # Args:
- # project_name (str) - None > default project,
- # scope (list of utils.EditableScope)
- # (optional, None is all scopes, default is LOCAL)
- #
- # Returns:
- # (dict of list of dict)
- # {
- # siteA : [
- # {
- # key:"root", label:"root",
- # "type": "dict",
- # "children":[
- # { "key": "work",
- # "type": "text",
- # "value": "c:/projects"}
- # ]
- # },
- # {
- # key:"credentials_url", label:"Credentials url",
- # "value":"'c:/projects/cred.json'", "type": "text",
- # "namespace": "{project_setting}/global/sync_server/
- # sites"
- # }
- # ]
- # }
- # """
- # allowed_sites = set()
- # sites = self.get_all_site_configs(project_name)
- # if project_name:
- # # Local Settings can select only from allowed sites for project
- # allowed_sites.update(set(self.get_active_sites(project_name)))
- # allowed_sites.update(set(self.get_remote_sites(project_name)))
- #
- # editable = {}
- # for site_name in sites.keys():
- # if allowed_sites and site_name not in allowed_sites:
- # continue
- #
- # items = self.get_configurable_items_for_site(project_name,
- # site_name,
- # scope)
- # # Local Settings need 'local' instead of real value
- # site_name = site_name.replace(get_local_site_id(), 'local')
- # editable[site_name] = items
- #
- # return editable
- #
- # def get_configurable_items_for_site(self, project_name=None,
- # site_name=None,
- # scope=None):
- # """
- # Returns list of items that could be configurable.
- #
- # Args:
- # project_name (str) - None > default project
- # site_name (str)
- # scope (list of utils.EditableScope)
- # (optional, None is all scopes)
- #
- # Returns:
- # (list)
- # [
- # {
- # key:"root", label:"root", type:"dict",
- # "children":[
- # { "key": "work",
- # "type": "text",
- # "value": "c:/projects"}
- # ]
- # }, ...
- # ]
- # """
- # provider_name = self.get_provider_for_site(site=site_name)
- # items = lib.factory.get_provider_configurable_items(provider_name)
- #
- # if project_name:
- # sync_s = self.get_sync_project_setting(project_name,
- # exclude_locals=True,
- # cached=False)
- # else:
- # sync_s = get_default_project_settings(exclude_locals=True)
- # sync_s = sync_s["global"]["sync_server"]
- # sync_s["sites"].update(
- # self._get_default_site_configs(self.enabled))
- #
- # editable = []
- # if type(scope) is not list:
- # scope = [scope]
- # scope = set(scope)
- # for key, properties in items.items():
- # if scope is None or scope.intersection(set(properties["scope"])):
- # val = sync_s.get("sites", {}).get(site_name, {}).get(key)
- #
- # item = {
- # "key": key,
- # "label": properties["label"],
- # "type": properties["type"]
- # }
- #
- # if properties.get("namespace"):
- # item["namespace"] = properties.get("namespace")
- # if "platform" in item["namespace"]:
- # try:
- # if val:
- # val = val[platform.system().lower()]
- # except KeyError:
- # st = "{}'s field value {} should be".format(key, val) # noqa: E501
- # log.error(st + " multiplatform dict")
- #
- # item["namespace"] = item["namespace"].replace('{site}',
- # site_name)
- # children = []
- # if properties["type"] == "dict":
- # if val:
- # for val_key, val_val in val.items():
- # child = {
- # "type": "text",
- # "key": val_key,
- # "value": val_val
- # }
- # children.append(child)
- #
- # if properties["type"] == "dict":
- # item["children"] = children
- # else:
- # item["value"] = val
- #
- # editable.append(item)
- #
- # return editable
-
- def reset_timer(self):
- """
- Called when waiting for next loop should be skipped.
-
- In case of user's involvement (reset site), start that right away.
- """
- self.sync_server_thread.reset_timer()
-
- def get_enabled_projects(self):
- """Returns list of projects which have SyncServer enabled."""
- enabled_projects = []
-
- if self.enabled:
- for project in self.connection.projects(projection={"name": 1}):
- project_name = project["name"]
- project_settings = self.get_sync_project_setting(project_name)
- if project_settings and project_settings.get("enabled"):
- enabled_projects.append(project_name)
-
- return enabled_projects
-
- def handle_alternate_site(self, collection, representation, processed_site,
- file_id, synced_file_id):
- """
- For special use cases where one site vendors another.
-
- Current use case is sftp site vendoring (exposing) same data as
- regular site (studio). Each site is accessible for different
- audience. 'studio' for artists in a studio, 'sftp' for externals.
-
- Change of file status on one site actually means same change on
- 'alternate' site. (eg. artists publish to 'studio', 'sftp' is using
- same location >> file is accesible on 'sftp' site right away.
-
- Args:
- collection (str): name of project
- representation (dict)
- processed_site (str): real site_name of published/uploaded file
- file_id (ObjectId): DB id of file handled
- synced_file_id (str): id of the created file returned
- by provider
- """
- sites = self.sync_system_settings.get("sites", {})
- sites[self.DEFAULT_SITE] = {"provider": "local_drive",
- "alternative_sites": []}
-
- alternate_sites = []
- for site_name, site_info in sites.items():
- conf_alternative_sites = site_info.get("alternative_sites", [])
- if processed_site in conf_alternative_sites:
- alternate_sites.append(site_name)
- continue
- if processed_site == site_name and conf_alternative_sites:
- alternate_sites.extend(conf_alternative_sites)
- continue
-
- alternate_sites = set(alternate_sites)
-
- for alt_site in alternate_sites:
- query = {
- "_id": representation["_id"]
- }
- elem = {"name": alt_site,
- "created_dt": datetime.now(),
- "id": synced_file_id}
-
- self.log.debug("Adding alternate {} to {}".format(
- alt_site, representation["_id"]))
- self._add_site(collection, query,
- [representation], elem,
- alt_site, file_id=file_id, force=True)
-
- """ End of Public API """
-
- def get_local_file_path(self, collection, site_name, file_path):
- """
- Externalized for app
- """
- handler = LocalDriveHandler(collection, site_name)
- local_file_path = handler.resolve_path(file_path)
-
- return local_file_path
-
- def _get_remote_sites_from_settings(self, sync_settings):
- if not self.enabled or not sync_settings.get('enabled'):
- return []
-
- remote_sites = [self.DEFAULT_SITE, self.LOCAL_SITE]
- if sync_settings:
- remote_sites.extend(sync_settings.get("sites").keys())
-
- return list(set(remote_sites))
-
- def _get_enabled_sites_from_settings(self, sync_settings):
- sites = [self.DEFAULT_SITE]
- if self.enabled and sync_settings.get('enabled'):
- sites.append(self.LOCAL_SITE)
-
- return sites
-
- def tray_init(self):
- """
- Actual initialization of Sync Server for Tray.
-
- Called when tray is initialized, it checks if module should be
- enabled. If not, no initialization necessary.
- """
- self.server_init()
-
- from .tray.app import SyncServerWindow
- self.widget = SyncServerWindow(self)
-
- def server_init(self):
- """Actual initialization of Sync Server."""
- # import only in tray or Python3, because of Python2 hosts
- from .sync_server import SyncServerThread
-
- if not self.enabled:
- return
-
- enabled_projects = self.get_enabled_projects()
- if not enabled_projects:
- self.enabled = False
- return
-
- self.lock = threading.Lock()
-
- self.sync_server_thread = SyncServerThread(self)
-
- def tray_start(self):
- """
- Triggered when Tray is started.
-
- Checks if configuration presets are available and if there is
- any provider ('gdrive', 'S3') that is activated
- (eg. has valid credentials).
-
- Returns:
- None
- """
- self.server_start()
-
- def server_start(self):
- if self.sync_project_settings and self.enabled:
- self.sync_server_thread.start()
- else:
- log.info("No presets or active providers. " +
- "Synchronization not possible.")
-
- def tray_exit(self):
- """
- Stops sync thread if running.
-
- Called from Module Manager
- """
- self.server_exit()
-
- def server_exit(self):
- if not self.sync_server_thread:
- return
-
- if not self.is_running:
- return
- try:
- log.info("Stopping sync server server")
- self.sync_server_thread.is_running = False
- self.sync_server_thread.stop()
- log.info("Sync server stopped")
- except Exception:
- log.warning(
- "Error has happened during Killing sync server",
- exc_info=True
- )
-
- def tray_menu(self, parent_menu):
- if not self.enabled:
- return
-
- from Qt import QtWidgets
- """Add menu or action to Tray(or parent)'s menu"""
- action = QtWidgets.QAction(self.label, parent_menu)
- action.triggered.connect(self.show_widget)
- parent_menu.addAction(action)
- parent_menu.addSeparator()
-
- self.action_show_widget = action
-
- @property
- def is_running(self):
- return self.sync_server_thread.is_running
-
- def get_anatomy(self, project_name):
- """
- Get already created or newly created anatomy for project
-
- Args:
- project_name (string):
-
- Return:
- (Anatomy)
- """
- return self._anatomies.get('project_name') or Anatomy(project_name)
-
- @property
- def connection(self):
- if self._connection is None:
- self._connection = AvalonMongoDB()
-
- return self._connection
-
- @property
- def sync_system_settings(self):
- if self._sync_system_settings is None:
- self._sync_system_settings = get_system_settings()["modules"].\
- get("sync_server")
-
- return self._sync_system_settings
-
- @property
- def sync_project_settings(self):
- if self._sync_project_settings is None:
- self.set_sync_project_settings()
-
- return self._sync_project_settings
-
- def set_sync_project_settings(self, exclude_locals=False):
- """
- Set sync_project_settings for all projects (caching)
- Args:
- exclude_locals (bool): ignore overrides from Local Settings
- For performance
- """
- sync_project_settings = self._prepare_sync_project_settings(
- exclude_locals)
-
- self._sync_project_settings = sync_project_settings
-
- def _prepare_sync_project_settings(self, exclude_locals):
- sync_project_settings = {}
- system_sites = self.get_all_site_configs()
- project_docs = self.connection.projects(
- projection={"name": 1},
- only_active=True
- )
- for project_doc in project_docs:
- project_name = project_doc["name"]
- sites = copy.deepcopy(system_sites) # get all configured sites
- proj_settings = self._parse_sync_settings_from_settings(
- get_project_settings(project_name,
- exclude_locals=exclude_locals))
- sites.update(self._get_default_site_configs(
- proj_settings["enabled"], project_name))
- sites.update(proj_settings['sites'])
- proj_settings["sites"] = sites
-
- sync_project_settings[project_name] = proj_settings
- if not sync_project_settings:
- log.info("No enabled and configured projects for sync.")
- return sync_project_settings
-
- def get_sync_project_setting(self, project_name, exclude_locals=False,
- cached=True):
- """ Handles pulling sync_server's settings for enabled 'project_name'
-
- Args:
- project_name (str): used in project settings
- exclude_locals (bool): ignore overrides from Local Settings
- cached (bool): use pre-cached values, or return fresh ones
- cached values needed for single loop (with all overrides)
- fresh values needed for Local settings (without overrides)
- Returns:
- (dict): settings dictionary for the enabled project,
- empty if no settings or sync is disabled
- """
- # presets set already, do not call again and again
- # self.log.debug("project preset {}".format(self.presets))
- if not cached:
- return self._prepare_sync_project_settings(exclude_locals)\
- [project_name]
-
- if not self.sync_project_settings or \
- not self.sync_project_settings.get(project_name):
- self.set_sync_project_settings(exclude_locals)
-
- return self.sync_project_settings.get(project_name)
-
- def _parse_sync_settings_from_settings(self, settings):
- """ settings from api.get_project_settings, TOOD rename """
- sync_settings = settings.get("global").get("sync_server")
-
- return sync_settings
-
- def get_all_site_configs(self, project_name=None):
- """
- Returns (dict) with all sites configured system wide.
-
- Args:
- project_name (str)(optional): if present, check if not disabled
-
- Returns:
- (dict): {'studio': {'provider':'local_drive'...},
- 'MY_LOCAL': {'provider':....}}
- """
- sync_sett = self.sync_system_settings
- project_enabled = True
- if project_name:
- project_enabled = project_name in self.get_enabled_projects()
- sync_enabled = sync_sett["enabled"] and project_enabled
-
- system_sites = {}
- if sync_enabled:
- for site, detail in sync_sett.get("sites", {}).items():
- system_sites[site] = detail
-
- system_sites.update(self._get_default_site_configs(sync_enabled,
- project_name))
-
- return system_sites
-
- def _get_default_site_configs(self, sync_enabled=True, project_name=None):
- """
- Returns settings for 'studio' and user's local site
-
- Returns base values from setting, not overriden by Local Settings,
- eg. value used to push TO LS not to get actual value for syncing.
- """
- if not project_name:
- anatomy_sett = get_default_anatomy_settings(exclude_locals=True)
- else:
- anatomy_sett = get_anatomy_settings(project_name,
- exclude_locals=True)
- roots = {}
- for root, config in anatomy_sett["roots"].items():
- roots[root] = config[platform.system().lower()]
- studio_config = {
- 'provider': 'local_drive',
- "root": roots
- }
- all_sites = {self.DEFAULT_SITE: studio_config}
- if sync_enabled:
- all_sites[get_local_site_id()] = {'provider': 'local_drive'}
- return all_sites
-
- def get_provider_for_site(self, project_name=None, site=None):
- """
- Return provider name for site (unique name across all projects.
- """
- sites = {self.DEFAULT_SITE: "local_drive",
- self.LOCAL_SITE: "local_drive",
- get_local_site_id(): "local_drive"}
-
- if site in sites.keys():
- return sites[site]
-
- if project_name: # backward compatibility
- proj_settings = self.get_sync_project_setting(project_name)
- provider = proj_settings.get("sites", {}).get(site, {}).\
- get("provider")
- if provider:
- return provider
-
- sync_sett = self.sync_system_settings
- for conf_site, detail in sync_sett.get("sites", {}).items():
- sites[conf_site] = detail.get("provider")
-
- return sites.get(site, 'N/A')
-
- @time_function
- def get_sync_representations(self, collection, active_site, remote_site):
- """
- Get representations that should be synced, these could be
- recognised by presence of document in 'files.sites', where key is
- a provider (GDrive, S3) and value is empty document or document
- without 'created_dt' field. (Don't put null to 'created_dt'!).
-
- Querying of 'to-be-synched' files is offloaded to Mongod for
- better performance. Goal is to get as few representations as
- possible.
- Args:
- collection (string): name of collection (in most cases matches
- project name
- active_site (string): identifier of current active site (could be
- 'local_0' when working from home, 'studio' when working in the
- studio (default)
- remote_site (string): identifier of remote site I want to sync to
-
- Returns:
- (list) of dictionaries
- """
- log.debug("Check representations for : {}".format(collection))
- self.connection.Session["AVALON_PROJECT"] = collection
- # retry_cnt - number of attempts to sync specific file before giving up
- retries_arr = self._get_retries_arr(collection)
- match = {
- "type": "representation",
- "$or": [
- {"$and": [
- {
- "files.sites": {
- "$elemMatch": {
- "name": active_site,
- "created_dt": {"$exists": True}
- }
- }}, {
- "files.sites": {
- "$elemMatch": {
- "name": {"$in": [remote_site]},
- "created_dt": {"$exists": False},
- "tries": {"$in": retries_arr}
- }
- }
- }]},
- {"$and": [
- {
- "files.sites": {
- "$elemMatch": {
- "name": active_site,
- "created_dt": {"$exists": False},
- "tries": {"$in": retries_arr}
- }
- }}, {
- "files.sites": {
- "$elemMatch": {
- "name": {"$in": [remote_site]},
- "created_dt": {"$exists": True}
- }
- }
- }
- ]}
- ]
- }
-
- aggr = [
- {"$match": match},
- {'$unwind': '$files'},
- {'$addFields': {
- 'order_remote': {
- '$filter': {'input': '$files.sites', 'as': 'p',
- 'cond': {'$eq': ['$$p.name', remote_site]}
- }},
- 'order_local': {
- '$filter': {'input': '$files.sites', 'as': 'p',
- 'cond': {'$eq': ['$$p.name', active_site]}
- }},
- }},
- {'$addFields': {
- 'priority': {
- '$cond': [
- {'$size': '$order_local.priority'},
- {'$first': '$order_local.priority'},
- {'$cond': [
- {'$size': '$order_remote.priority'},
- {'$first': '$order_remote.priority'},
- self.DEFAULT_PRIORITY]}
- ]
- },
- }},
- {'$group': {
- '_id': '$_id',
- # pass through context - same for representation
- 'context': {'$addToSet': '$context'},
- 'data': {'$addToSet': '$data'},
- # pass through files as a list
- 'files': {'$addToSet': '$files'},
- 'priority': {'$max': "$priority"},
- }},
- {"$sort": {'priority': -1, '_id': 1}},
- ]
- log.debug("active_site:{} - remote_site:{}".format(active_site,
- remote_site))
- log.debug("query: {}".format(aggr))
- representations = self.connection.aggregate(aggr)
-
- return representations
-
- def check_status(self, file, local_site, remote_site, config_preset):
- """
- Check synchronization status for single 'file' of single
- 'representation' by single 'provider'.
- (Eg. check if 'scene.ma' of lookdev.v10 should be synced to GDrive
-
- Always is comparing local record, eg. site with
- 'name' == self.presets[PROJECT_NAME]['config']["active_site"]
-
- This leads to trigger actual upload or download, there is
- a use case 'studio' <> 'remote' where user should publish
- to 'studio', see progress in Tray GUI, but do not do
- physical upload/download
- (as multiple user would be doing that).
-
- Do physical U/D only when any of the sites is user's local, in that
- case only user has the data and must U/D.
-
- Args:
- file (dictionary): of file from representation in Mongo
- local_site (string): - local side of compare (usually 'studio')
- remote_site (string): - gdrive etc.
- config_preset (dict): config about active site, retries
- Returns:
- (string) - one of SyncStatus
- """
- sites = file.get("sites") or []
-
- if get_local_site_id() not in (local_site, remote_site):
- # don't do upload/download for studio sites
- log.debug("No local site {} - {}".format(local_site, remote_site))
- return SyncStatus.DO_NOTHING
-
- _, remote_rec = self._get_site_rec(sites, remote_site) or {}
- if remote_rec: # sync remote target
- created_dt = remote_rec.get("created_dt")
- if not created_dt:
- tries = self._get_tries_count_from_rec(remote_rec)
- # file will be skipped if unsuccessfully tried over threshold
- # error metadata needs to be purged manually in DB to reset
- if tries < int(config_preset["retry_cnt"]):
- return SyncStatus.DO_UPLOAD
- else:
- _, local_rec = self._get_site_rec(sites, local_site) or {}
- if not local_rec or not local_rec.get("created_dt"):
- tries = self._get_tries_count_from_rec(local_rec)
- # file will be skipped if unsuccessfully tried over
- # threshold times, error metadata needs to be purged
- # manually in DB to reset
- if tries < int(config_preset["retry_cnt"]):
- return SyncStatus.DO_DOWNLOAD
-
- return SyncStatus.DO_NOTHING
-
- def update_db(self, collection, new_file_id, file, representation,
- site, error=None, progress=None, priority=None):
- """
- Update 'provider' portion of records in DB with success (file_id)
- or error (exception)
-
- Args:
- collection (string): name of project - force to db connection as
- each file might come from different collection
- new_file_id (string):
- file (dictionary): info about processed file (pulled from DB)
- representation (dictionary): parent repr of file (from DB)
- site (string): label ('gdrive', 'S3')
- error (string): exception message
- progress (float): 0-1 of progress of upload/download
- priority (int): 0-100 set priority
-
- Returns:
- None
- """
- representation_id = representation.get("_id")
- file_id = None
- if file:
- file_id = file.get("_id")
-
- query = {
- "_id": representation_id
- }
-
- update = {}
- if new_file_id:
- update["$set"] = self._get_success_dict(new_file_id)
- # reset previous errors if any
- update["$unset"] = self._get_error_dict("", "", "")
- elif progress is not None:
- update["$set"] = self._get_progress_dict(progress)
- elif priority is not None:
- update["$set"] = self._get_priority_dict(priority, file_id)
- else:
- tries = self._get_tries_count(file, site)
- tries += 1
-
- update["$set"] = self._get_error_dict(error, tries)
-
- arr_filter = [
- {'s.name': site}
- ]
- if file_id:
- arr_filter.append({'f._id': ObjectId(file_id)})
-
- self.connection.database[collection].update_one(
- query,
- update,
- upsert=True,
- array_filters=arr_filter
- )
-
- if progress is not None or priority is not None:
- return
-
- status = 'failed'
- error_str = 'with error {}'.format(error)
- if new_file_id:
- status = 'succeeded with id {}'.format(new_file_id)
- error_str = ''
-
- source_file = file.get("path", "")
- log.debug("File for {} - {source_file} process {status} {error_str}".
- format(representation_id,
- status=status,
- source_file=source_file,
- error_str=error_str))
-
- def _get_file_info(self, files, _id):
- """
- Return record from list of records which name matches to 'provider'
- Could be possibly refactored with '_get_provider_rec' together.
-
- Args:
- files (list): of dictionaries with info about published files
- _id (string): _id of specific file
-
- Returns:
- (int, dictionary): index from list and record with metadata
- about site (if/when created, errors..)
- OR (-1, None) if not present
- """
- for index, rec in enumerate(files):
- if rec.get("_id") == _id:
- return index, rec
-
- return -1, None
-
- def _get_site_rec(self, sites, site_name):
- """
- Return record from list of records which name matches to
- 'remote_site_name'
-
- Args:
- sites (list): of dictionaries
- site_name (string): 'local_XXX', 'gdrive'
-
- Returns:
- (int, dictionary): index from list and record with metadata
- about site (if/when created, errors..)
- OR (-1, None) if not present
- """
- for index, rec in enumerate(sites):
- if rec.get("name") == site_name:
- return index, rec
-
- return -1, None
-
- def reset_site_on_representation(self, collection, representation_id,
- side=None, file_id=None, site_name=None,
- remove=False, pause=None, force=False):
- """
- Reset information about synchronization for particular 'file_id'
- and provider.
- Useful for testing or forcing file to be reuploaded.
-
- 'side' and 'site_name' are disjunctive.
-
- 'side' is used for resetting local or remote side for
- current user for repre.
-
- 'site_name' is used to set synchronization for particular site.
- Should be used when repre should be synced to new site.
-
- Args:
- collection (string): name of project (eg. collection) in DB
- representation_id(string): _id of representation
- file_id (string): file _id in representation
- side (string): local or remote side
- site_name (string): for adding new site
- remove (bool): if True remove site altogether
- pause (bool or None): if True - pause, False - unpause
- force (bool): hard reset - currently only for add_site
-
- Returns:
- throws ValueError
- """
- query = {
- "_id": ObjectId(representation_id)
- }
-
- representation = list(self.connection.database[collection].find(query))
- if not representation:
- raise ValueError("Representation {} not found in {}".
- format(representation_id, collection))
- if side and site_name:
- raise ValueError("Misconfiguration, only one of side and " +
- "site_name arguments should be passed.")
-
- local_site = self.get_active_site(collection)
- remote_site = self.get_remote_site(collection)
-
- if side:
- if side == 'local':
- site_name = local_site
- else:
- site_name = remote_site
-
- elem = {"name": site_name}
-
- if file_id: # reset site for particular file
- self._reset_site_for_file(collection, query,
- elem, file_id, site_name)
- elif side: # reset site for whole representation
- self._reset_site(collection, query, elem, site_name)
- elif remove: # remove site for whole representation
- self._remove_site(collection, query, representation, site_name)
- elif pause is not None:
- self._pause_unpause_site(collection, query,
- representation, site_name, pause)
- else: # add new site to all files for representation
- self._add_site(collection, query, representation, elem, site_name,
- force)
-
- def _update_site(self, collection, query, update, arr_filter):
- """
- Auxiliary method to call update_one function on DB
-
- Used for refactoring ugly reset_provider_for_file
- """
- self.connection.database[collection].update_one(
- query,
- update,
- upsert=True,
- array_filters=arr_filter
- )
-
- def _reset_site_for_file(self, collection, query,
- elem, file_id, site_name):
- """
- Resets 'site_name' for 'file_id' on representation in 'query' on
- 'collection'
- """
- update = {
- "$set": {"files.$[f].sites.$[s]": elem}
- }
- if not isinstance(file_id, ObjectId):
- file_id = ObjectId(file_id)
-
- arr_filter = [
- {'s.name': site_name},
- {'f._id': file_id}
- ]
-
- self._update_site(collection, query, update, arr_filter)
-
- def _reset_site(self, collection, query, elem, site_name):
- """
- Resets 'site_name' for all files of representation in 'query'
- """
- update = {
- "$set": {"files.$[].sites.$[s]": elem}
- }
-
- arr_filter = [
- {'s.name': site_name}
- ]
-
- self._update_site(collection, query, update, arr_filter)
-
- def _remove_site(self, collection, query, representation, site_name):
- """
- Removes 'site_name' for 'representation' in 'query'
-
- Throws ValueError if 'site_name' not found on 'representation'
- """
- found = False
- for repre_file in representation.pop().get("files"):
- for site in repre_file.get("sites"):
- if site.get("name") == site_name:
- found = True
- break
- if not found:
- msg = "Site {} not found".format(site_name)
- log.info(msg)
- raise ValueError(msg)
-
- update = {
- "$pull": {"files.$[].sites": {"name": site_name}}
- }
- arr_filter = []
-
- self._update_site(collection, query, update, arr_filter)
-
- def _pause_unpause_site(self, collection, query,
- representation, site_name, pause):
- """
- Pauses/unpauses all files for 'representation' based on 'pause'
-
- Throws ValueError if 'site_name' not found on 'representation'
- """
- found = False
- site = None
- for repre_file in representation.pop().get("files"):
- for site in repre_file.get("sites"):
- if site["name"] == site_name:
- found = True
- break
- if not found:
- msg = "Site {} not found".format(site_name)
- log.info(msg)
- raise ValueError(msg)
-
- if pause:
- site['paused'] = pause
- else:
- if site.get('paused'):
- site.pop('paused')
-
- update = {
- "$set": {"files.$[].sites.$[s]": site}
- }
-
- arr_filter = [
- {'s.name': site_name}
- ]
-
- self._update_site(collection, query, update, arr_filter)
-
- def _add_site(self, collection, query, representation, elem, site_name,
- force=False, file_id=None):
- """
- Adds 'site_name' to 'representation' on 'collection'
-
- Args:
- representation (list of 1 dict)
- file_id (ObjectId)
-
- Use 'force' to remove existing or raises ValueError
- """
- reseted_existing = False
- for repre_file in representation.pop().get("files"):
- if file_id and file_id != repre_file["_id"]:
- continue
-
- for site in repre_file.get("sites"):
- if site["name"] == site_name:
- if force:
- self._reset_site_for_file(collection, query,
- elem, repre_file["_id"],
- site_name)
- reseted_existing = True
- else:
- msg = "Site {} already present".format(site_name)
- log.info(msg)
- raise ValueError(msg)
-
- if reseted_existing:
- return
-
- if not file_id:
- update = {
- "$push": {"files.$[].sites": elem}
- }
-
- arr_filter = []
- else:
- update = {
- "$push": {"files.$[f].sites": elem}
- }
- arr_filter = [
- {'f._id': file_id}
- ]
-
- self._update_site(collection, query, update, arr_filter)
-
- def _remove_local_file(self, collection, representation_id, site_name):
- """
- Removes all local files for 'site_name' of 'representation_id'
-
- Args:
- collection (string): project name (must match DB)
- representation_id (string): MongoDB _id value
- site_name (string): name of configured and active site
-
- Returns:
- only logs, catches IndexError and OSError
- """
- my_local_site = get_local_site_id()
- if my_local_site != site_name:
- self.log.warning("Cannot remove non local file for {}".
- format(site_name))
- return
-
- provider_name = self.get_provider_for_site(site=site_name)
-
- if provider_name == 'local_drive':
- query = {
- "_id": ObjectId(representation_id)
- }
-
- representation = list(
- self.connection.database[collection].find(query))
- if not representation:
- self.log.debug("No repre {} found".format(
- representation_id))
- return
-
- representation = representation.pop()
- local_file_path = ''
- for file in representation.get("files"):
- local_file_path = self.get_local_file_path(collection,
- site_name,
- file.get("path", "")
- )
- try:
- self.log.debug("Removing {}".format(local_file_path))
- os.remove(local_file_path)
- except IndexError:
- msg = "No file set for {}".format(representation_id)
- self.log.debug(msg)
- raise ValueError(msg)
- except OSError:
- msg = "File {} cannot be removed".format(file["path"])
- self.log.warning(msg)
- raise ValueError(msg)
-
- folder = None
- try:
- folder = os.path.dirname(local_file_path)
- os.rmdir(folder)
- except OSError:
- msg = "folder {} cannot be removed".format(folder)
- self.log.warning(msg)
- raise ValueError(msg)
-
- def get_loop_delay(self, project_name):
- """
- Return count of seconds before next synchronization loop starts
- after finish of previous loop.
- Returns:
- (int): in seconds
- """
- ld = self.sync_project_settings[project_name]["config"]["loop_delay"]
- return int(ld)
-
- def show_widget(self):
- """Show dialog for Sync Queue"""
- no_errors = False
- try:
- from .tray.app import SyncServerWindow
- self.widget = SyncServerWindow(self)
- no_errors = True
- except ValueError:
- log.info("No system setting for sync. Not syncing.", exc_info=True)
- except KeyError:
- log.info((
- "There are not set presets for SyncServer OR "
- "Credentials provided are invalid, "
- "no syncing possible").
- format(str(self.sync_project_settings)), exc_info=True)
- except:
- log.error("Uncaught exception durin start of SyncServer",
- exc_info=True)
- self.enabled = no_errors
- self.widget.show()
-
- def _get_success_dict(self, new_file_id):
- """
- Provide success metadata ("id", "created_dt") to be stored in Db.
- Used in $set: "DICT" part of query.
- Sites are array inside of array(file), so real indexes for both
- file and site are needed for upgrade in DB.
- Args:
- new_file_id: id of created file
- Returns:
- (dictionary)
- """
- val = {"files.$[f].sites.$[s].id": new_file_id,
- "files.$[f].sites.$[s].created_dt": datetime.now()}
- return val
-
- def _get_error_dict(self, error="", tries="", progress=""):
- """
- Provide error metadata to be stored in Db.
- Used for set (error and tries provided) or unset mode.
- Args:
- error: (string) - message
- tries: how many times failed
- Returns:
- (dictionary)
- """
- val = {"files.$[f].sites.$[s].last_failed_dt": datetime.now(),
- "files.$[f].sites.$[s].error": error,
- "files.$[f].sites.$[s].tries": tries,
- "files.$[f].sites.$[s].progress": progress
- }
- return val
-
- def _get_tries_count_from_rec(self, rec):
- """
- Get number of failed attempts to sync from site record
- Args:
- rec (dictionary): info about specific site record
- Returns:
- (int) - number of failed attempts
- """
- if not rec:
- return 0
- return rec.get("tries", 0)
-
- def _get_tries_count(self, file, provider):
- """
- Get number of failed attempts to sync
- Args:
- file (dictionary): info about specific file
- provider (string): name of site ('gdrive' or specific user site)
- Returns:
- (int) - number of failed attempts
- """
- _, rec = self._get_site_rec(file.get("sites", []), provider)
- return rec.get("tries", 0)
-
- def _get_progress_dict(self, progress):
- """
- Provide progress metadata to be stored in Db.
- Used during upload/download for GUI to show.
- Args:
- progress: (float) - 0-1 progress of upload/download
- Returns:
- (dictionary)
- """
- val = {"files.$[f].sites.$[s].progress": progress}
- return val
-
- def _get_priority_dict(self, priority, file_id):
- """
- Provide priority metadata to be stored in Db.
- Used during upload/download for GUI to show.
- Args:
- priority: (int) - priority for file(s)
- Returns:
- (dictionary)
- """
- if file_id:
- str_key = "files.$[f].sites.$[s].priority"
- else:
- str_key = "files.$[].sites.$[s].priority"
- return {str_key: int(priority)}
-
- def _get_retries_arr(self, project_name):
- """
- Returns array with allowed values in 'tries' field. If repre
- contains these values, it means it was tried to be synchronized
- but failed. We try up to 'self.presets["retry_cnt"]' times before
- giving up and skipping representation.
- Returns:
- (list)
- """
- retry_cnt = self.sync_project_settings[project_name].\
- get("config")["retry_cnt"]
- arr = [i for i in range(int(retry_cnt))]
- arr.append(None)
-
- return arr
-
- def _get_roots_config(self, presets, project_name, site_name):
- """
- Returns configured root(s) for 'project_name' and 'site_name' from
- settings ('presets')
- """
- return presets[project_name]['sites'][site_name]['root']
diff --git a/openpype/modules/default_modules/sync_server/tray/app.py b/openpype/modules/default_modules/sync_server/tray/app.py
deleted file mode 100644
index fc8558bdbcf..00000000000
--- a/openpype/modules/default_modules/sync_server/tray/app.py
+++ /dev/null
@@ -1,149 +0,0 @@
-from Qt import QtWidgets, QtCore, QtGui
-
-from openpype.tools.settings import style
-
-from openpype.lib import PypeLogger
-from openpype import resources
-
-from .widgets import (
- SyncProjectListWidget,
- SyncRepresentationSummaryWidget
-)
-
-log = PypeLogger().get_logger("SyncServer")
-
-
-class SyncServerWindow(QtWidgets.QDialog):
- """
- Main window that contains list of synchronizable projects and summary
- view with all synchronizable representations for first project
- """
-
- def __init__(self, sync_server, parent=None):
- super(SyncServerWindow, self).__init__(parent)
- self.sync_server = sync_server
- self.setWindowFlags(QtCore.Qt.Window)
- self.setFocusPolicy(QtCore.Qt.StrongFocus)
-
- self.setStyleSheet(style.load_stylesheet())
- self.setWindowIcon(QtGui.QIcon(resources.get_openpype_icon_filepath()))
- self.resize(1450, 700)
-
- self.timer = QtCore.QTimer()
- self.timer.timeout.connect(self._hide_message)
-
- body = QtWidgets.QWidget(self)
- footer = QtWidgets.QWidget(self)
- footer.setFixedHeight(20)
-
- left_column = QtWidgets.QWidget(body)
- left_column_layout = QtWidgets.QVBoxLayout(left_column)
-
- self.projects = SyncProjectListWidget(sync_server, self)
- self.projects.refresh() # force selection of default
- left_column_layout.addWidget(self.projects)
- self.pause_btn = QtWidgets.QPushButton("Pause server")
-
- left_column_layout.addWidget(self.pause_btn)
-
- repres = SyncRepresentationSummaryWidget(
- sync_server,
- project=self.projects.current_project,
- parent=self)
- container = QtWidgets.QWidget()
- container_layout = QtWidgets.QHBoxLayout(container)
- container_layout.setContentsMargins(0, 0, 0, 0)
- split = QtWidgets.QSplitter()
- split.addWidget(left_column)
- split.addWidget(repres)
- split.setSizes([180, 950, 200])
- container_layout.addWidget(split)
-
- body_layout = QtWidgets.QHBoxLayout(body)
- body_layout.addWidget(container)
- body_layout.setContentsMargins(0, 0, 0, 0)
-
- self.message = QtWidgets.QLabel(footer)
- self.message.hide()
-
- footer_layout = QtWidgets.QVBoxLayout(footer)
- footer_layout.addWidget(self.message)
- footer_layout.setContentsMargins(20, 0, 0, 0)
-
- layout = QtWidgets.QVBoxLayout(self)
- layout.addWidget(body)
- layout.addWidget(footer)
-
- self.setWindowTitle("Sync Queue")
-
- self.projects.project_changed.connect(
- self._on_project_change
- )
-
- self.pause_btn.clicked.connect(self._pause)
- self.pause_btn.setAutoDefault(False)
- self.pause_btn.setDefault(False)
- repres.message_generated.connect(self._update_message)
- self.projects.message_generated.connect(self._update_message)
-
- self.representationWidget = repres
-
- def _on_project_change(self):
- if self.projects.current_project is None:
- return
-
- self.representationWidget.table_view.model().set_project(
- self.projects.current_project
- )
-
- project_name = self.projects.current_project
- if not self.sync_server.get_sync_project_setting(project_name):
- self.projects.message_generated.emit(
- "Project {} not active anymore".format(project_name))
- self.projects.refresh()
- return
-
- def showEvent(self, event):
- self.representationWidget.model.set_project(
- self.projects.current_project)
- self.projects.refresh()
- self._set_running(True)
- super().showEvent(event)
-
- def closeEvent(self, event):
- self._set_running(False)
- super().closeEvent(event)
-
- def _set_running(self, running):
- self.representationWidget.model.is_running = running
- self.representationWidget.model.timer.setInterval(0)
-
- def _pause(self):
- if self.sync_server.is_paused():
- self.sync_server.unpause_server()
- self.pause_btn.setText("Pause server")
- else:
- self.sync_server.pause_server()
- self.pause_btn.setText("Unpause server")
- self.projects.refresh()
-
- def _update_message(self, value):
- """
- Update and show message in the footer
- """
- self.message.setText(value)
- if self.message.isVisible():
- self.message.repaint()
- else:
- self.message.show()
- msec_delay = 3000
- self.timer.start(msec_delay)
-
- def _hide_message(self):
- """
- Hide message in footer
-
- Called automatically by self.timer after a while
- """
- self.message.setText("")
- self.message.hide()
diff --git a/openpype/modules/default_modules/sync_server/tray/delegates.py b/openpype/modules/default_modules/sync_server/tray/delegates.py
deleted file mode 100644
index 5ab809a8166..00000000000
--- a/openpype/modules/default_modules/sync_server/tray/delegates.py
+++ /dev/null
@@ -1,139 +0,0 @@
-import os
-from Qt import QtCore, QtWidgets, QtGui
-
-from openpype.lib import PypeLogger
-from . import lib
-
-from openpype.tools.utils.constants import (
- LOCAL_PROVIDER_ROLE,
- REMOTE_PROVIDER_ROLE,
- LOCAL_PROGRESS_ROLE,
- REMOTE_PROGRESS_ROLE,
- LOCAL_DATE_ROLE,
- REMOTE_DATE_ROLE,
- LOCAL_FAILED_ROLE,
- REMOTE_FAILED_ROLE,
- EDIT_ICON_ROLE
-)
-
-log = PypeLogger().get_logger("SyncServer")
-
-
-class PriorityDelegate(QtWidgets.QStyledItemDelegate):
- """Creates editable line edit to set priority on representation"""
- def paint(self, painter, option, index):
- super(PriorityDelegate, self).paint(painter, option, index)
-
- if option.widget.selectionModel().isSelected(index) or \
- option.state & QtWidgets.QStyle.State_MouseOver:
- edit_icon = index.data(EDIT_ICON_ROLE)
- if not edit_icon:
- return
-
- state = QtGui.QIcon.On
- mode = QtGui.QIcon.Selected
-
- icon_side = 16
- icon_rect = QtCore.QRect(
- option.rect.left() + option.rect.width() - icon_side - 4,
- option.rect.top() + ((option.rect.height() - icon_side) / 2),
- icon_side,
- icon_side
- )
-
- edit_icon.paint(
- painter, icon_rect,
- QtCore.Qt.AlignRight, mode, state
- )
-
- def createEditor(self, parent, option, index):
- editor = PriorityLineEdit(
- parent,
- option.widget.selectionModel().selectedRows())
- editor.setFocus()
- return editor
-
- def setModelData(self, editor, model, index):
- for index in editor.selected_idxs:
- try:
- val = int(editor.text())
- except ValueError:
- val = model.sync_server.DEFAULT_PRIORITY
- model.set_priority_data(index, val)
-
-
-class PriorityLineEdit(QtWidgets.QLineEdit):
- """Special LineEdit to consume Enter and store selected indexes"""
- def __init__(self, parent=None, selected_idxs=None):
- self.selected_idxs = selected_idxs
- super(PriorityLineEdit, self).__init__(parent)
-
- def keyPressEvent(self, event):
- result = super(PriorityLineEdit, self).keyPressEvent(event)
- if (
- event.key() in (QtCore.Qt.Key_Return, QtCore.Qt.Key_Enter)
- ):
- return event.accept()
-
- return result
-
-
-class ImageDelegate(QtWidgets.QStyledItemDelegate):
- """
- Prints icon of site and progress of synchronization
- """
-
- def __init__(self, parent=None, side=None):
- super(ImageDelegate, self).__init__(parent)
- self.icons = {}
- self.side = side
-
- def paint(self, painter, option, index):
- super(ImageDelegate, self).paint(painter, option, index)
- option = QtWidgets.QStyleOptionViewItem(option)
- option.showDecorationSelected = True
-
- if not self.side:
- log.warning("No side provided, delegate won't work")
- return
-
- if self.side == 'local':
- provider = index.data(LOCAL_PROVIDER_ROLE)
- value = index.data(LOCAL_PROGRESS_ROLE)
- date_value = index.data(LOCAL_DATE_ROLE)
- is_failed = index.data(LOCAL_FAILED_ROLE)
- else:
- provider = index.data(REMOTE_PROVIDER_ROLE)
- value = index.data(REMOTE_PROGRESS_ROLE)
- date_value = index.data(REMOTE_DATE_ROLE)
- is_failed = index.data(REMOTE_FAILED_ROLE)
-
- if not self.icons.get(provider):
- resource_path = os.path.dirname(__file__)
- resource_path = os.path.join(resource_path, "..",
- "providers", "resources")
- pix_url = "{}/{}.png".format(resource_path, provider)
- pixmap = QtGui.QPixmap(pix_url)
- self.icons[provider] = pixmap
- else:
- pixmap = self.icons[provider]
-
- padding = 10
- point = QtCore.QPoint(option.rect.x() + padding,
- option.rect.y() +
- (option.rect.height() - pixmap.height()) / 2)
- painter.drawPixmap(point, pixmap)
-
- overlay_rect = option.rect.translated(0, 0)
- overlay_rect.setHeight(overlay_rect.height() * (1.0 - float(value)))
- painter.fillRect(overlay_rect,
- QtGui.QBrush(QtGui.QColor(0, 0, 0, 100)))
- text_rect = option.rect.translated(10, 0)
- painter.drawText(text_rect,
- QtCore.Qt.AlignCenter,
- date_value)
-
- if is_failed:
- overlay_rect = option.rect.translated(0, 0)
- painter.fillRect(overlay_rect,
- QtGui.QBrush(QtGui.QColor(255, 0, 0, 35)))
diff --git a/openpype/modules/default_modules/sync_server/tray/lib.py b/openpype/modules/default_modules/sync_server/tray/lib.py
deleted file mode 100644
index 87344be634d..00000000000
--- a/openpype/modules/default_modules/sync_server/tray/lib.py
+++ /dev/null
@@ -1,158 +0,0 @@
-import attr
-import abc
-import six
-
-from openpype.lib import PypeLogger
-
-
-log = PypeLogger().get_logger("SyncServer")
-
-STATUS = {
- 0: 'In Progress',
- 1: 'Queued',
- 2: 'Failed',
- 3: 'Paused',
- 4: 'Synced OK',
- -1: 'Not available'
-}
-
-DUMMY_PROJECT = "No project configured"
-
-
-@six.add_metaclass(abc.ABCMeta)
-class AbstractColumnFilter:
-
- def __init__(self, column_name, dbcon=None):
- self.column_name = column_name
- self.dbcon = dbcon
- self._search_variants = []
-
- def search_variants(self):
- """
- Returns all flavors of search available for this column,
- """
- return self._search_variants
-
- @abc.abstractmethod
- def values(self):
- """
- Returns dict of available values for filter {'label':'value'}
- """
- pass
-
- @abc.abstractmethod
- def prepare_match_part(self, values):
- """
- Prepares format valid for $match part from 'values
-
- Args:
- values (dict): {'label': 'value'}
- Returns:
- (dict): {'COLUMN_NAME': {'$in': ['val1', 'val2']}}
- """
- pass
-
-
-class PredefinedSetFilter(AbstractColumnFilter):
-
- def __init__(self, column_name, values):
- super().__init__(column_name)
- self._search_variants = ['checkbox']
- self._values = values
- if self._values and \
- list(self._values.keys())[0] == list(self._values.values())[0]:
- self._search_variants.append('text')
-
- def values(self):
- return {k: v for k, v in self._values.items()}
-
- def prepare_match_part(self, values):
- return {'$in': list(values.keys())}
-
-
-class RegexTextFilter(AbstractColumnFilter):
-
- def __init__(self, column_name):
- super().__init__(column_name)
- self._search_variants = ['text']
-
- def values(self):
- return {}
-
- def prepare_match_part(self, values):
- """ values = {'text1 text2': 'text1 text2'} """
- if not values:
- return {}
-
- regex_strs = set()
- text = list(values.keys())[0] # only single key always expected
- for word in text.split():
- regex_strs.add('.*{}.*'.format(word))
-
- return {"$regex": "|".join(regex_strs),
- "$options": 'i'}
-
-
-class MultiSelectFilter(AbstractColumnFilter):
-
- def __init__(self, column_name, values=None, dbcon=None):
- super().__init__(column_name)
- self._values = values
- self.dbcon = dbcon
- self._search_variants = ['checkbox']
-
- def values(self):
- if self._values:
- return {k: v for k, v in self._values.items()}
-
- recs = self.dbcon.find({'type': self.column_name}, {"name": 1,
- "_id": -1})
- values = {}
- for item in recs:
- values[item["name"]] = item["name"]
- return dict(sorted(values.items(), key=lambda it: it[1]))
-
- def prepare_match_part(self, values):
- return {'$in': list(values.keys())}
-
-
-@attr.s
-class FilterDefinition:
- type = attr.ib()
- values = attr.ib(factory=list)
-
-
-def pretty_size(value, suffix='B'):
- for unit in ['', 'Ki', 'Mi', 'Gi', 'Ti', 'Pi', 'Ei', 'Zi']:
- if abs(value) < 1024.0:
- return "%3.1f%s%s" % (value, unit, suffix)
- value /= 1024.0
- return "%.1f%s%s" % (value, 'Yi', suffix)
-
-
-def convert_progress(value):
- try:
- progress = float(value)
- except (ValueError, TypeError):
- progress = 0.0
-
- return progress
-
-
-def translate_provider_for_icon(sync_server, project, site):
- """
- Get provider for 'site'
-
- This is used for getting icon, 'studio' should have different icon
- then local sites, even the provider 'local_drive' is same
-
- """
- if site == sync_server.DEFAULT_SITE:
- return sync_server.DEFAULT_SITE
- return sync_server.get_provider_for_site(site=site)
-
-
-def get_value_from_id_by_role(model, object_id, role):
- """Return value from item with 'object_id' with 'role'."""
- index = model.get_index(object_id)
- return model.data(index, role)
diff --git a/openpype/modules/default_modules/sync_server/tray/models.py b/openpype/modules/default_modules/sync_server/tray/models.py
deleted file mode 100644
index 80f41992cb3..00000000000
--- a/openpype/modules/default_modules/sync_server/tray/models.py
+++ /dev/null
@@ -1,1359 +0,0 @@
-import os
-import attr
-from bson.objectid import ObjectId
-
-from Qt import QtCore
-from Qt.QtCore import Qt
-
-from openpype.tools.utils.delegates import pretty_timestamp
-from avalon.vendor import qtawesome
-
-from openpype.lib import PypeLogger
-from openpype.api import get_local_site_id
-
-from . import lib
-
-from openpype.tools.utils.constants import (
- LOCAL_PROVIDER_ROLE,
- REMOTE_PROVIDER_ROLE,
- LOCAL_PROGRESS_ROLE,
- REMOTE_PROGRESS_ROLE,
- HEADER_NAME_ROLE,
- EDIT_ICON_ROLE,
- LOCAL_DATE_ROLE,
- REMOTE_DATE_ROLE,
- LOCAL_FAILED_ROLE,
- REMOTE_FAILED_ROLE,
- STATUS_ROLE,
- PATH_ROLE,
- ERROR_ROLE,
- TRIES_ROLE
-)
-
-
-log = PypeLogger().get_logger("SyncServer")
-
-
-class _SyncRepresentationModel(QtCore.QAbstractTableModel):
-
- COLUMN_LABELS = []
-
- PAGE_SIZE = 20 # default page size to query for
- REFRESH_SEC = 5000 # in seconds, requery DB for new status
-
- refresh_started = QtCore.Signal()
- refresh_finished = QtCore.Signal()
-
- @property
- def dbcon(self):
- """
- Database object with preselected project (collection) to run DB
- operations (find, aggregate).
-
- All queries should go through this (because of collection).
- """
- return self.sync_server.connection.database[self.project]
-
- @property
- def project(self):
- """Returns project"""
- return self._project
-
- @property
- def column_filtering(self):
- return self._column_filtering
-
- @property
- def is_running(self):
- return self._is_running
-
- @is_running.setter
- def is_running(self, state):
- self._is_running = state
-
- def rowCount(self, _index):
- return len(self._data)
-
- def columnCount(self, _index=None):
- return len(self._header)
-
- def headerData(self, section, orientation, role=Qt.DisplayRole):
- if section >= len(self.COLUMN_LABELS):
- return
-
- if role == Qt.DisplayRole:
- if orientation == Qt.Horizontal:
- return self.COLUMN_LABELS[section][1]
-
- if role == HEADER_NAME_ROLE:
- if orientation == Qt.Horizontal:
- return self.COLUMN_LABELS[section][0] # return name
-
- def data(self, index, role):
- item = self._data[index.row()]
-
- header_value = self._header[index.column()]
- if role == LOCAL_PROVIDER_ROLE:
- return item.local_provider
-
- if role == REMOTE_PROVIDER_ROLE:
- return item.remote_provider
-
- if role == LOCAL_PROGRESS_ROLE:
- return item.local_progress
-
- if role == REMOTE_PROGRESS_ROLE:
- return item.remote_progress
-
- if role == LOCAL_DATE_ROLE:
- if item.created_dt:
- return pretty_timestamp(item.created_dt)
-
- if role == REMOTE_DATE_ROLE:
- if item.sync_dt:
- return pretty_timestamp(item.sync_dt)
-
- if role == LOCAL_FAILED_ROLE:
- return item.status == lib.STATUS[2] and \
- item.local_progress < 1
-
- if role == REMOTE_FAILED_ROLE:
- return item.status == lib.STATUS[2] and \
- item.remote_progress < 1
-
- if role in (Qt.DisplayRole, Qt.EditRole):
- # because of ImageDelegate
- if header_value in ['remote_site', 'local_site']:
- return ""
-
- return attr.asdict(item)[self._header[index.column()]]
-
- if role == EDIT_ICON_ROLE:
- if self.can_edit and header_value in self.EDITABLE_COLUMNS:
- return self.edit_icon
-
- if role == PATH_ROLE:
- return item.path
-
- if role == ERROR_ROLE:
- return item.error
-
- if role == TRIES_ROLE:
- return item.tries
-
- if role == STATUS_ROLE:
- return item.status
-
- if role == Qt.UserRole:
- return item._id
-
- @property
- def can_edit(self):
- """Returns true if some site is user local site, eg. could edit"""
- return get_local_site_id() in (self.active_site, self.remote_site)
-
- def get_column(self, index):
- """
- Returns info about column
-
- Args:
- index (QModelIndex)
- Returns:
- (tuple): (COLUMN_NAME: COLUMN_LABEL)
- """
- return self.COLUMN_LABELS[index]
-
- def get_header_index(self, value):
- """
- Returns index of 'value' in headers
-
- Args:
- value (str): header name value
- Returns:
- (int)
- """
- return self._header.index(value)
-
- def refresh(self, representations=None, load_records=0):
- """
- Reloads representations from DB if necessary, adds them to model.
-
- Runs periodically (every X seconds) or by demand (change of
- sorting, filtering etc.)
-
- Emits 'modelReset' signal.
-
- Args:
- representations (PaginationResult object): pass result of
- aggregate query from outside - mostly for testing only
- load_records (int) - enforces how many records should be
- actually queried (scrolled a couple of times to list more
- than single page of records)
- """
- if self.is_editing or not self.is_running:
- return
- self.refresh_started.emit()
- self.beginResetModel()
- self._data = []
- self._rec_loaded = 0
-
- if not representations:
- self.query = self.get_query(load_records)
- representations = self.dbcon.aggregate(pipeline=self.query,
- allowDiskUse=True)
-
- self.add_page_records(self.active_site, self.remote_site,
- representations)
- self.endResetModel()
- self.refresh_finished.emit()
-
- def tick(self):
- """
- Triggers refresh of model.
-
- Because of pagination, prepared (sorting, filtering) query needs
- to be run on DB every X seconds.
- """
- self.refresh(representations=None, load_records=self._rec_loaded)
- self.timer.start(self.REFRESH_SEC)
-
- def canFetchMore(self, _index):
- """
- Check if there are more records than currently loaded
- """
- # 'skip' might be suboptimal when representation hits 500k+
- return self._total_records > self._rec_loaded
-
- def fetchMore(self, index):
- """
- Add more record to model.
-
- Called when 'canFetchMore' returns true, which means there are
- more records in DB than loaded.
- """
- log.debug("fetchMore")
- items_to_fetch = min(self._total_records - self._rec_loaded,
- self.PAGE_SIZE)
- self.query = self.get_query(self._rec_loaded)
- representations = self.dbcon.aggregate(pipeline=self.query,
- allowDiskUse=True)
- self.beginInsertRows(index,
- self._rec_loaded,
- self._rec_loaded + items_to_fetch - 1)
-
- self.add_page_records(self.active_site, self.remote_site,
- representations)
-
- self.endInsertRows()
-
- def sort(self, index, order):
- """
- Summary sort per representation.
-
- Sort is happening on a DB side, model is reset, db queried
- again.
-
- It remembers one last sort, adds it as secondary after new sort.
-
- Args:
- index (int): column index
- order (int): 0|
- """
- # limit unwanted first re-sorting by view
- if index < 0:
- return
-
- self._rec_loaded = 0
- if order == 0:
- order = 1
- else:
- order = -1
-
- backup_sort = dict(self.sort_criteria)
-
- self.sort_criteria = {self.SORT_BY_COLUMN[index]: order} # reset
- # add last one
- for key, val in backup_sort.items():
- if key != '_id' and key != self.SORT_BY_COLUMN[index]:
- self.sort_criteria[key] = val
- break
- # add default one
- self.sort_criteria['_id'] = 1
-
- self.query = self.get_query()
- # import json
- # log.debug(json.dumps(self.query, indent=4).\
- # replace('False', 'false').\
- # replace('True', 'true').replace('None', 'null'))
-
- representations = self.dbcon.aggregate(pipeline=self.query,
- allowDiskUse=True)
- self.refresh(representations)
-
- def set_word_filter(self, word_filter):
- """
- Adds text value filtering
-
- Args:
- word_filter (str): string inputted by user
- """
- self._word_filter = word_filter
- self.refresh()
-
- def get_filters(self):
- """
- Returns all available filter editors per column_name keys.
- """
- filters = {}
- for column_name, _ in self.COLUMN_LABELS:
- filter_rec = self.COLUMN_FILTERS.get(column_name)
- if filter_rec:
- filter_rec.dbcon = self.dbcon
- filters[column_name] = filter_rec
-
- return filters
-
- def get_column_filter(self, index):
- """
- Returns filter object for column 'index
-
- Args:
- index(int): index of column in header
-
- Returns:
- (AbstractColumnFilter)
- """
- column_name = self._header[index]
-
- filter_rec = self.COLUMN_FILTERS.get(column_name)
- if filter_rec:
- filter_rec.dbcon = self.dbcon # up-to-date db connection
-
- return filter_rec
-
- def set_column_filtering(self, checked_values):
- """
- Sets dictionary used in '$match' part of MongoDB aggregate
-
- Args:
- checked_values(dict): key:values ({'status':{1:"Foo",3:"Bar"}}
-
- Modifies:
- self._column_filtering : {'status': {'$in': [1, 2, 3]}}
- """
- filtering = {}
- for column_name, dict_value in checked_values.items():
- column_f = self.COLUMN_FILTERS.get(column_name)
- if not column_f:
- continue
- column_f.dbcon = self.dbcon
- filtering[column_name] = column_f.prepare_match_part(dict_value)
-
- self._column_filtering = filtering
-
- def get_column_filter_values(self, index):
- """
- Returns list of available values for filtering in the column
-
- Args:
- index(int): index of column in header
-
- Returns:
- (dict) of value: label shown in filtering menu
- 'value' is used in MongoDB query, 'label' is human readable for
- menu
- for some columns ('subset') might be 'value' and 'label' same
- """
- filter_rec = self.get_column_filter(index)
- if not filter_rec:
- return {}
-
- return filter_rec.values()
-
- def set_project(self, project):
- """
- Changes project, called after project selection is changed
-
- Args:
- project (str): name of project
- """
- self._project = project
- self.sync_server.set_sync_project_settings()
- # project might have been deactivated in the meantime
- if not self.sync_server.get_sync_project_setting(project):
- return
-
- self.active_site = self.sync_server.get_active_site(self.project)
- self.remote_site = self.sync_server.get_remote_site(self.project)
- self.refresh()
-
- def get_index(self, id):
- """
- Get index of 'id' value.
-
- Used for keeping selection after refresh.
-
- Args:
- id (str): MongoDB _id
- Returns:
- (QModelIndex)
- """
- for i in range(self.rowCount(None)):
- index = self.index(i, 0)
- value = self.data(index, Qt.UserRole)
- if value == id:
- return index
- return None
-
-
-class SyncRepresentationSummaryModel(_SyncRepresentationModel):
- """
- Model for summary of representations.
-
- Groups files information per representation. Allows sorting and
- full text filtering.
-
- Allows pagination, most of heavy lifting is being done on DB side.
- Single model matches to single collection. When project is changed,
- model is reset and refreshed.
-
- Args:
- sync_server (SyncServer) - object to call server operations (update
- db status, set site status...)
- header (list) - names of visible columns
- project (string) - collection name, all queries must be called on
- a specific collection
-
- """
- COLUMN_LABELS = [
- ("asset", "Asset"),
- ("subset", "Subset"),
- ("version", "Version"),
- ("representation", "Representation"),
- ("local_site", "Active site"),
- ("remote_site", "Remote site"),
- ("files_count", "Files"),
- ("files_size", "Size"),
- ("priority", "Priority"),
- ("status", "Status")
- ]
-
- DEFAULT_SORT = {
- "updated_dt_remote": -1,
- "_id": 1
- }
- SORT_BY_COLUMN = [
- "asset", # asset
- "subset", # subset
- "version", # version
- "representation", # representation
- "updated_dt_local", # local created_dt
- "updated_dt_remote", # remote created_dt
- "files_count", # count of files
- "files_size", # file size of all files
- "priority", # priority
- "status" # status
- ]
-
- COLUMN_FILTERS = {
- 'status': lib.PredefinedSetFilter('status', lib.STATUS),
- 'subset': lib.RegexTextFilter('subset'),
- 'asset': lib.RegexTextFilter('asset'),
- 'representation': lib.MultiSelectFilter('representation')
- }
-
- EDITABLE_COLUMNS = ["priority"]
-
- refresh_started = QtCore.Signal()
- refresh_finished = QtCore.Signal()
-
- @attr.s
- class SyncRepresentation:
- """
- Auxiliary object for easier handling.
-
- Fields must contain all header values (+ any arbitrary values).
- """
- _id = attr.ib()
- asset = attr.ib()
- subset = attr.ib()
- version = attr.ib()
- representation = attr.ib()
- created_dt = attr.ib(default=None)
- sync_dt = attr.ib(default=None)
- local_site = attr.ib(default=None)
- remote_site = attr.ib(default=None)
- local_provider = attr.ib(default=None)
- remote_provider = attr.ib(default=None)
- local_progress = attr.ib(default=None)
- remote_progress = attr.ib(default=None)
- files_count = attr.ib(default=None)
- files_size = attr.ib(default=None)
- priority = attr.ib(default=None)
- status = attr.ib(default=None)
- path = attr.ib(default=None)
-
- def __init__(self, sync_server, header, project=None, parent=None):
- super(SyncRepresentationSummaryModel, self).__init__(parent=parent)
- self._header = header
- self._data = []
- self._project = project
- self._rec_loaded = 0
- self._total_records = 0 # how many documents query actually found
- self._word_filter = None
- self._column_filtering = {}
- self._is_running = False
-
- self.edit_icon = qtawesome.icon("fa.edit", color="white")
- self.is_editing = False
-
- self._word_filter = None
-
- if not self._project or self._project == lib.DUMMY_PROJECT:
- return
-
- self.sync_server = sync_server
- # TODO think about admin mode
- # this is for regular user, always only single local and single remote
- self.active_site = self.sync_server.get_active_site(self.project)
- self.remote_site = self.sync_server.get_remote_site(self.project)
-
- self.sort_criteria = self.DEFAULT_SORT
-
- self.query = self.get_query()
- self.default_query = list(self.get_query())
-
- representations = self.dbcon.aggregate(pipeline=self.query,
- allowDiskUse=True)
- self.refresh(representations)
-
- self.timer = QtCore.QTimer()
- self.timer.timeout.connect(self.tick)
- self.timer.start(self.REFRESH_SEC)
-
- def add_page_records(self, local_site, remote_site, representations):
- """
- Process all records from 'representation' and add them to storage.
-
- Args:
- local_site (str): name of local site (mine)
- remote_site (str): name of cloud provider (theirs)
- representations (Mongo Cursor) - mimics result set, 1 object
- with paginatedResults array and totalCount array
- """
- result = representations.next()
- count = 0
- total_count = result.get("totalCount")
- if total_count:
- count = total_count.pop().get('count')
- self._total_records = count
-
- local_provider = lib.translate_provider_for_icon(self.sync_server,
- self.project,
- local_site)
- remote_provider = lib.translate_provider_for_icon(self.sync_server,
- self.project,
- remote_site)
-
- for repre in result.get("paginatedResults"):
- files = repre.get("files", [])
- if isinstance(files, dict): # aggregate returns dictionary
- files = [files]
-
- # representation without files doesnt concern us
- if not files:
- continue
-
- local_updated = remote_updated = None
- if repre.get('updated_dt_local'):
- local_updated = \
- repre.get('updated_dt_local').strftime("%Y%m%dT%H%M%SZ")
-
- if repre.get('updated_dt_remote'):
- remote_updated = \
- repre.get('updated_dt_remote').strftime("%Y%m%dT%H%M%SZ")
-
- avg_progress_remote = lib.convert_progress(
- repre.get('avg_progress_remote', '0'))
- avg_progress_local = lib.convert_progress(
- repre.get('avg_progress_local', '0'))
-
- if repre.get("version"):
- version = "v{:0>3d}".format(repre.get("version"))
- else:
- version = "master"
-
- item = self.SyncRepresentation(
- repre.get("_id"),
- repre.get("asset"),
- repre.get("subset"),
- version,
- repre.get("representation"),
- local_updated,
- remote_updated,
- local_site,
- remote_site,
- local_provider,
- remote_provider,
- avg_progress_local,
- avg_progress_remote,
- repre.get("files_count", 1),
- lib.pretty_size(repre.get("files_size", 0)),
- repre.get("priority"),
- lib.STATUS[repre.get("status", -1)],
- files[0].get('path')
- )
-
- self._data.append(item)
- self._rec_loaded += 1
-
- def get_query(self, limit=0):
- """
- Returns basic aggregate query for main table.
-
- Main table provides summary information about representation,
- which could have multiple files. Details are accessible after
- double click on representation row.
- Columns:
- 'created_dt' - max of created or updated (when failed) per repr
- 'sync_dt' - same for remote side
- 'local_site' - progress of repr on local side, 1 = finished
- 'remote_site' - progress on remote side, calculates from files
- 'status' -
- 0 - in progress
- 1 - failed
- 2 - queued
- 3 - paused
- 4 - finished on both sides
-
- are calculated and must be calculated in DB because of
- pagination
-
- Args:
- limit (int): how many records should be returned, by default
- it 'PAGE_SIZE' for performance.
- Should be overridden by value of loaded records for refresh
- functionality (got more records by scrolling, refresh
- shouldn't reset that)
- """
- if limit == 0:
- limit = SyncRepresentationSummaryModel.PAGE_SIZE
-
- aggr = [
- {"$match": self.get_match_part()},
- {'$unwind': '$files'},
- # merge potentially unwinded records back to single per repre
- {'$addFields': {
- 'order_remote': {
- '$filter': {'input': '$files.sites', 'as': 'p',
- 'cond': {'$eq': ['$$p.name', self.remote_site]}
- }},
- 'order_local': {
- '$filter': {'input': '$files.sites', 'as': 'p',
- 'cond': {'$eq': ['$$p.name', self.active_site]}
- }}
- }},
- {'$addFields': {
- # prepare progress per file, presence of 'created_dt' denotes
- # successfully finished load/download
- 'progress_remote': {'$first': {
- '$cond': [{'$size': "$order_remote.progress"},
- "$order_remote.progress",
- {'$cond': [
- {'$size': "$order_remote.created_dt"},
- [1],
- [0]
- ]}
- ]}},
- 'progress_local': {'$first': {
- '$cond': [{'$size': "$order_local.progress"},
- "$order_local.progress",
- {'$cond': [
- {'$size': "$order_local.created_dt"},
- [1],
- [0]
- ]}
- ]}},
- # file might be successfully created or failed, not both
- 'updated_dt_remote': {'$first': {
- '$cond': [{'$size': "$order_remote.created_dt"},
- "$order_remote.created_dt",
- {'$cond': [
- {'$size': "$order_remote.last_failed_dt"},
- "$order_remote.last_failed_dt",
- []
- ]}
- ]}},
- 'updated_dt_local': {'$first': {
- '$cond': [{'$size': "$order_local.created_dt"},
- "$order_local.created_dt",
- {'$cond': [
- {'$size': "$order_local.last_failed_dt"},
- "$order_local.last_failed_dt",
- []
- ]}
- ]}},
- 'files_size': {'$ifNull': ["$files.size", 0]},
- 'failed_remote': {
- '$cond': [{'$size': "$order_remote.last_failed_dt"},
- 1,
- 0]},
- 'failed_local': {
- '$cond': [{'$size': "$order_local.last_failed_dt"},
- 1,
- 0]},
- 'failed_local_tries': {
- '$cond': [{'$size': '$order_local.tries'},
- {'$first': '$order_local.tries'},
- 0]},
- 'failed_remote_tries': {
- '$cond': [{'$size': '$order_remote.tries'},
- {'$first': '$order_remote.tries'},
- 0]},
- 'paused_remote': {
- '$cond': [{'$size': "$order_remote.paused"},
- 1,
- 0]},
- 'paused_local': {
- '$cond': [{'$size': "$order_local.paused"},
- 1,
- 0]},
- 'priority': {
- '$cond': [
- {'$size': '$order_local.priority'},
- {'$first': '$order_local.priority'},
- {'$cond': [
- {'$size': '$order_remote.priority'},
- {'$first': '$order_remote.priority'},
- self.sync_server.DEFAULT_PRIORITY]}
- ]
- },
- }},
- {'$group': {
- '_id': '$_id',
- # pass through context - same for representation
- 'context': {'$addToSet': '$context'},
- 'data': {'$addToSet': '$data'},
- # pass through files as a list
- 'files': {'$addToSet': '$files'},
- # count how many files
- 'files_count': {'$sum': 1},
- 'files_size': {'$sum': '$files_size'},
- # sum avg progress, finished = 1
- 'avg_progress_remote': {'$avg': "$progress_remote"},
- 'avg_progress_local': {'$avg': "$progress_local"},
- # select last touch of file
- 'updated_dt_remote': {'$max': "$updated_dt_remote"},
- 'failed_remote': {'$sum': '$failed_remote'},
- 'failed_local': {'$sum': '$failed_local'},
- 'failed_remote_tries': {'$sum': '$failed_remote_tries'},
- 'failed_local_tries': {'$sum': '$failed_local_tries'},
- 'paused_remote': {'$sum': '$paused_remote'},
- 'paused_local': {'$sum': '$paused_local'},
- 'updated_dt_local': {'$max': "$updated_dt_local"},
- 'priority': {'$max': "$priority"},
- }},
- {"$project": self.projection}
- ]
-
- if self.column_filtering:
- aggr.append(
- {"$match": self.column_filtering}
- )
-
- aggr.extend(
- [{"$sort": self.sort_criteria},
- {
- '$facet': {
- 'paginatedResults': [{'$skip': self._rec_loaded},
- {'$limit': limit}],
- 'totalCount': [{'$count': 'count'}]
- }
- }]
- )
-
- return aggr
-
- def get_match_part(self):
- """
- Extend match part with word_filter if present.
-
- Filter is set by user input. Each model has different fields to be
- checked.
- If performance issues are found, '$text' and text indexes should
- be investigated.
-
- Fulltext searches in:
- context.subset
- context.asset
- context.representation names AND _id (ObjectId)
- """
- base_match = {
- "type": "representation",
- 'files.sites.name': {'$all': [self.active_site,
- self.remote_site]}
- }
- if not self._word_filter:
- return base_match
- else:
- regex_str = '.*{}.*'.format(self._word_filter)
- base_match['$or'] = [
- {'context.subset': {'$regex': regex_str, '$options': 'i'}},
- {'context.asset': {'$regex': regex_str, '$options': 'i'}},
- {'context.representation': {'$regex': regex_str,
- '$options': 'i'}}]
-
- if ObjectId.is_valid(self._word_filter):
- base_match['$or'] = [{'_id': ObjectId(self._word_filter)}]
-
- return base_match
-
- @property
- def projection(self):
- """
- Projection part for aggregate query.
-
- All fields with '1' will be returned, no others.
-
- Returns:
- (dict)
- """
- return {
- "subset": {"$first": "$context.subset"},
- "asset": {"$first": "$context.asset"},
- "version": {"$first": "$context.version"},
- "representation": {"$first": "$context.representation"},
- "data.path": 1,
- "files": 1,
- 'files_count': 1,
- "files_size": 1,
- 'avg_progress_remote': 1,
- 'avg_progress_local': 1,
- 'updated_dt_remote': 1,
- 'updated_dt_local': 1,
- 'paused_remote': 1,
- 'paused_local': 1,
- 'priority': 1,
- 'status': {
- '$switch': {
- 'branches': [
- {
- 'case': {
- '$or': ['$paused_remote', '$paused_local']},
- 'then': 3 # Paused
- },
- {
- 'case': {
- '$or': [
- {'$gte': ['$failed_local_tries', 3]},
- {'$gte': ['$failed_remote_tries', 3]}
- ]},
- 'then': 2}, # Failed
- {
- 'case': {
- '$or': [{'$eq': ['$avg_progress_remote', 0]},
- {'$eq': ['$avg_progress_local', 0]}]},
- 'then': 1 # Queued
- },
- {
- 'case': {'$or': [{'$and': [
- {'$gt': ['$avg_progress_remote', 0]},
- {'$lt': ['$avg_progress_remote', 1]}
- ]},
- {'$and': [
- {'$gt': ['$avg_progress_local', 0]},
- {'$lt': ['$avg_progress_local', 1]}
- ]}
- ]},
- 'then': 0 # In progress
- },
- {
- 'case': {'$and': [
- {'$eq': ['$avg_progress_remote', 1]},
- {'$eq': ['$avg_progress_local', 1]}
- ]},
- 'then': 4 # Synced OK
- },
- ],
- 'default': -1
- }
- }
- }
-
- def set_priority_data(self, index, value):
- """
- Sets 'priority' flag and value on local site for selected reprs.
-
- Args:
- index (QItemIndex): selected index from View
- value (int): priority value
-
- Updates DB.
- Potentially should allow set priority to any site when user
- management is implemented.
- """
- if not self.can_edit:
- return
-
- repre_id = self.data(index, Qt.UserRole)
-
- representation = list(self.dbcon.find({"type": "representation",
- "_id": repre_id}))
- if representation:
- self.sync_server.update_db(self.project, None, None,
- representation.pop(),
- get_local_site_id(),
- priority=value)
- self.is_editing = False
-
- # all other approaches messed up selection to 0th index
- self.timer.setInterval(0)
-
-
-class SyncRepresentationDetailModel(_SyncRepresentationModel):
- """
- List of all syncronizable files per single representation.
-
- Used in detail window accessible after clicking on single repre in the
- summary.
-
- Args:
- sync_server (SyncServer) - object to call server operations (update
- db status, set site status...)
- header (list) - names of visible columns
- _id (string) - MongoDB _id of representation
- project (string) - collection name, all queries must be called on
- a specific collection
- """
- COLUMN_LABELS = [
- ("file", "File name"),
- ("local_site", "Active site"),
- ("remote_site", "Remote site"),
- ("files_size", "Size"),
- ("priority", "Priority"),
- ("status", "Status")
- ]
-
- PAGE_SIZE = 30
- DEFAULT_SORT = {
- "files.path": 1
- }
- SORT_BY_COLUMN = [
- "files.path",
- "updated_dt_local", # local created_dt
- "updated_dt_remote", # remote created_dt
- "size", # remote progress
- "priority", # priority
- "status" # status
- ]
-
- COLUMN_FILTERS = {
- 'status': lib.PredefinedSetFilter('status', lib.STATUS),
- 'file': lib.RegexTextFilter('file'),
- }
-
- EDITABLE_COLUMNS = ["priority"]
-
- @attr.s
- class SyncRepresentationDetail:
- """
- Auxiliary object for easier handling.
-
- Fields must contain all header values (+ any arbitrary values).
- """
- _id = attr.ib()
- file = attr.ib()
- created_dt = attr.ib(default=None)
- sync_dt = attr.ib(default=None)
- local_site = attr.ib(default=None)
- remote_site = attr.ib(default=None)
- local_provider = attr.ib(default=None)
- remote_provider = attr.ib(default=None)
- local_progress = attr.ib(default=None)
- remote_progress = attr.ib(default=None)
- size = attr.ib(default=None)
- priority = attr.ib(default=None)
- status = attr.ib(default=None)
- tries = attr.ib(default=None)
- error = attr.ib(default=None)
- path = attr.ib(default=None)
-
- def __init__(self, sync_server, header, _id,
- project=None):
- super(SyncRepresentationDetailModel, self).__init__()
- self._header = header
- self._data = []
- self._project = project
- self._rec_loaded = 0
- self._total_records = 0 # how many documents query actually found
- self._word_filter = None
- self._id = _id
- self._column_filtering = {}
- self._is_running = False
-
- self.is_editing = False
- self.edit_icon = qtawesome.icon("fa.edit", color="white")
-
- self.sync_server = sync_server
- # TODO think about admin mode
- # this is for regular user, always only single local and single remote
- self.active_site = self.sync_server.get_active_site(self.project)
- self.remote_site = self.sync_server.get_remote_site(self.project)
-
- self.sort_criteria = self.DEFAULT_SORT
-
- self.query = self.get_query()
- representations = self.dbcon.aggregate(pipeline=self.query,
- allowDiskUse=True)
- self.refresh(representations)
-
- self.timer = QtCore.QTimer()
- self.timer.timeout.connect(self.tick)
- self.timer.start(SyncRepresentationSummaryModel.REFRESH_SEC)
-
- def add_page_records(self, local_site, remote_site, representations):
- """
- Process all records from 'representation' and add them to storage.
-
- Args:
- local_site (str): name of local site (mine)
- remote_site (str): name of cloud provider (theirs)
- representations (Mongo Cursor) - mimics result set, 1 object
- with paginatedResults array and totalCount array
- """
- # representations is a Cursor, get first
- result = representations.next()
- count = 0
- total_count = result.get("totalCount")
- if total_count:
- count = total_count.pop().get('count')
- self._total_records = count
-
- local_provider = lib.translate_provider_for_icon(self.sync_server,
- self.project,
- local_site)
- remote_provider = lib.translate_provider_for_icon(self.sync_server,
- self.project,
- remote_site)
-
- for repre in result.get("paginatedResults"):
- # log.info("!!! repre:: {}".format(repre))
- files = repre.get("files", [])
- if isinstance(files, dict): # aggregate returns dictionary
- files = [files]
-
- for file in files:
- local_updated = remote_updated = None
- if repre.get('updated_dt_local'):
- local_updated = \
- repre.get('updated_dt_local').strftime(
- "%Y%m%dT%H%M%SZ")
-
- if repre.get('updated_dt_remote'):
- remote_updated = \
- repre.get('updated_dt_remote').strftime(
- "%Y%m%dT%H%M%SZ")
-
- remote_progress = lib.convert_progress(
- repre.get('progress_remote', '0'))
- local_progress = lib.convert_progress(
- repre.get('progress_local', '0'))
-
- errors = []
- if repre.get('failed_remote_error'):
- errors.append(repre.get('failed_remote_error'))
- if repre.get('failed_local_error'):
- errors.append(repre.get('failed_local_error'))
-
- item = self.SyncRepresentationDetail(
- file.get("_id"),
- os.path.basename(file["path"]),
- local_updated,
- remote_updated,
- local_site,
- remote_site,
- local_provider,
- remote_provider,
- local_progress,
- remote_progress,
- lib.pretty_size(file.get('size', 0)),
- repre.get("priority"),
- lib.STATUS[repre.get("status", -1)],
- repre.get("tries"),
- '\n'.join(errors),
- file.get('path')
-
- )
- self._data.append(item)
- self._rec_loaded += 1
-
- def get_query(self, limit=0):
- """
- Gets query that gets used when no extra sorting, filtering or
- projecting is needed.
-
- Called for basic table view.
-
- Returns:
- [(dict)] - list with single dict - appropriate for aggregate
- function for MongoDB
- """
- if limit == 0:
- limit = SyncRepresentationSummaryModel.PAGE_SIZE
-
- aggr = [
- {"$match": self.get_match_part()},
- {"$unwind": "$files"},
- {'$addFields': {
- 'order_remote': {
- '$filter': {'input': '$files.sites', 'as': 'p',
- 'cond': {'$eq': ['$$p.name', self.remote_site]}
- }},
- 'order_local': {
- '$filter': {'input': '$files.sites', 'as': 'p',
- 'cond': {'$eq': ['$$p.name', self.active_site]}
- }}
- }},
- {'$addFields': {
- # prepare progress per file, presence of 'created_dt' denotes
- # successfully finished load/download
- 'progress_remote': {'$first': {
- '$cond': [{'$size': "$order_remote.progress"},
- "$order_remote.progress",
- {'$cond': [
- {'$size': "$order_remote.created_dt"},
- [1],
- [0]
- ]}
- ]}},
- 'progress_local': {'$first': {
- '$cond': [{'$size': "$order_local.progress"},
- "$order_local.progress",
- {'$cond': [
- {'$size': "$order_local.created_dt"},
- [1],
- [0]
- ]}
- ]}},
- # file might be successfully created or failed, not both
- 'updated_dt_remote': {'$first': {
- '$cond': [
- {'$size': "$order_remote.created_dt"},
- "$order_remote.created_dt",
- {
- '$cond': [
- {'$size': "$order_remote.last_failed_dt"},
- "$order_remote.last_failed_dt",
- []
- ]
- }
- ]
- }},
- 'updated_dt_local': {'$first': {
- '$cond': [
- {'$size': "$order_local.created_dt"},
- "$order_local.created_dt",
- {
- '$cond': [
- {'$size': "$order_local.last_failed_dt"},
- "$order_local.last_failed_dt",
- []
- ]
- }
- ]
- }},
- 'paused_remote': {
- '$cond': [{'$size': "$order_remote.paused"},
- 1,
- 0]},
- 'paused_local': {
- '$cond': [{'$size': "$order_local.paused"},
- 1,
- 0]},
- 'failed_remote': {
- '$cond': [{'$size': "$order_remote.last_failed_dt"},
- 1,
- 0]},
- 'failed_local': {
- '$cond': [{'$size': "$order_local.last_failed_dt"},
- 1,
- 0]},
- 'failed_remote_error': {'$first': {
- '$cond': [{'$size': "$order_remote.error"},
- "$order_remote.error",
- [""]]}},
- 'failed_local_error': {'$first': {
- '$cond': [{'$size': "$order_local.error"},
- "$order_local.error",
- [""]]}},
- 'tries': {'$first': {
- '$cond': [
- {'$size': "$order_local.tries"},
- "$order_local.tries",
- {'$cond': [
- {'$size': "$order_remote.tries"},
- "$order_remote.tries",
- []
- ]}
- ]}},
- 'priority': {
- '$cond': [
- {'$size': '$order_local.priority'},
- {'$first': '$order_local.priority'},
- {'$cond': [
- {'$size': '$order_remote.priority'},
- {'$first': '$order_remote.priority'},
- self.sync_server.DEFAULT_PRIORITY]}
- ]
- },
- }},
- {"$project": self.projection}
- ]
-
- if self.column_filtering:
- aggr.append(
- {"$match": self.column_filtering}
- )
- print(self.column_filtering)
-
- aggr.extend([
- {"$sort": self.sort_criteria},
- {
- '$facet': {
- 'paginatedResults': [{'$skip': self._rec_loaded},
- {'$limit': limit}],
- 'totalCount': [{'$count': 'count'}]
- }
- }
- ])
-
- return aggr
-
- def get_match_part(self):
- """
- Returns different content for 'match' portion if filtering by
- name is present
-
- Returns:
- (dict)
- """
- if not self._word_filter:
- return {
- "type": "representation",
- "_id": self._id
- }
- else:
- regex_str = '.*{}.*'.format(self._word_filter)
- return {
- "type": "representation",
- "_id": self._id,
- '$or': [{'files.path': {'$regex': regex_str, '$options': 'i'}}]
- }
-
- @property
- def projection(self):
- """
- Projection part for aggregate query.
-
- All fields with '1' will be returned, no others.
-
- Returns:
- (dict)
- """
- return {
- "files": 1,
- 'progress_remote': 1,
- 'progress_local': 1,
- 'updated_dt_remote': 1,
- 'updated_dt_local': 1,
- 'paused_remote': 1,
- 'paused_local': 1,
- 'failed_remote_error': 1,
- 'failed_local_error': 1,
- 'tries': 1,
- 'priority': 1,
- 'status': {
- '$switch': {
- 'branches': [
- {
- 'case': {
- '$or': ['$paused_remote', '$paused_local']},
- 'then': 3 # Paused
- },
- {
- 'case': {
- '$and': [{'$or': ['$failed_remote',
- '$failed_local']},
- {'$eq': ['$tries', 3]}]},
- 'then': 2 # Failed (3 tries)
- },
- {
- 'case': {
- '$or': [{'$eq': ['$progress_remote', 0]},
- {'$eq': ['$progress_local', 0]}]},
- 'then': 1 # Queued
- },
- {
- 'case': {
- '$or': ['$failed_remote', '$failed_local']},
- 'then': 2 # Failed
- },
- {
- 'case': {'$or': [{'$and': [
- {'$gt': ['$progress_remote', 0]},
- {'$lt': ['$progress_remote', 1]}
- ]},
- {'$and': [
- {'$gt': ['$progress_local', 0]},
- {'$lt': ['$progress_local', 1]}
- ]}
- ]},
- 'then': 0 # In Progress
- },
- {
- 'case': {'$and': [
- {'$eq': ['$progress_remote', 1]},
- {'$eq': ['$progress_local', 1]}
- ]},
- 'then': 4 # Synced OK
- },
- ],
- 'default': -1
- }
- },
- 'data.path': 1
- }
-
- def set_priority_data(self, index, value):
- """
- Sets 'priority' flag and value on local site for selected reprs.
-
- Args:
- index (QItemIndex): selected index from View
- value (int): priority value
-
- Updates DB
- """
- if not self.can_edit:
- return
-
- file_id = self.data(index, Qt.UserRole)
-
- updated_file = None
- # conversion from cursor to list
- representations = list(self.dbcon.find({"type": "representation",
- "_id": self._id}))
-
- representation = representations.pop()
- for repre_file in representation["files"]:
- if repre_file["_id"] == file_id:
- updated_file = repre_file
- break
-
- if representation and updated_file:
- self.sync_server.update_db(self.project, None, updated_file,
- representation, get_local_site_id(),
- priority=value)
- self.is_editing = False
- # all other approaches messed up selection to 0th index
- self.timer.setInterval(0)
diff --git a/openpype/modules/default_modules/sync_server/tray/widgets.py b/openpype/modules/default_modules/sync_server/tray/widgets.py
deleted file mode 100644
index 18487b3d11f..00000000000
--- a/openpype/modules/default_modules/sync_server/tray/widgets.py
+++ /dev/null
@@ -1,1216 +0,0 @@
-import os
-import subprocess
-import sys
-from functools import partial
-
-from Qt import QtWidgets, QtCore, QtGui
-from Qt.QtCore import Qt
-
-from openpype.tools.settings import style
-
-from openpype.api import get_local_site_id
-from openpype.lib import PypeLogger
-
-from openpype.tools.utils.delegates import pretty_timestamp
-from avalon.vendor import qtawesome
-
-from .models import (
- SyncRepresentationSummaryModel,
- SyncRepresentationDetailModel
-)
-
-from . import lib
-from . import delegates
-
-from openpype.tools.utils.constants import (
- LOCAL_PROGRESS_ROLE,
- REMOTE_PROGRESS_ROLE,
- HEADER_NAME_ROLE,
- STATUS_ROLE,
- PATH_ROLE,
- LOCAL_SITE_NAME_ROLE,
- REMOTE_SITE_NAME_ROLE,
- LOCAL_DATE_ROLE,
- REMOTE_DATE_ROLE,
- ERROR_ROLE,
- TRIES_ROLE
-)
-
-log = PypeLogger().get_logger("SyncServer")
-
-
-class SyncProjectListWidget(QtWidgets.QWidget):
- """
- Lists all projects that are synchronized to choose from
- """
- project_changed = QtCore.Signal()
- message_generated = QtCore.Signal(str)
-
- refresh_msec = 10000
-
- def __init__(self, sync_server, parent):
- super(SyncProjectListWidget, self).__init__(parent)
- self.setObjectName("ProjectListWidget")
-
- self._parent = parent
-
- label_widget = QtWidgets.QLabel("Projects", self)
- project_list = QtWidgets.QListView(self)
- project_model = QtGui.QStandardItemModel()
- project_list.setModel(project_model)
- project_list.setContextMenuPolicy(QtCore.Qt.CustomContextMenu)
-
- # Do not allow editing
- project_list.setEditTriggers(
- QtWidgets.QAbstractItemView.EditTrigger.NoEditTriggers
- )
-
- layout = QtWidgets.QVBoxLayout(self)
- layout.setContentsMargins(0, 0, 0, 0)
- layout.setSpacing(3)
- layout.addWidget(label_widget, 0)
- layout.addWidget(project_list, 1)
-
- project_list.customContextMenuRequested.connect(self._on_context_menu)
- project_list.selectionModel().selectionChanged.connect(
- self._on_selection_changed
- )
-
- self.project_model = project_model
- self.project_list = project_list
- self.sync_server = sync_server
- self.current_project = None
- self.project_name = None
- self.local_site = None
- self.remote_site = None
- self.icons = {}
-
- self._selection_changed = False
- self._model_reset = False
-
- timer = QtCore.QTimer()
- timer.setInterval(self.refresh_msec)
- timer.timeout.connect(self.refresh)
- timer.start()
-
- self.timer = timer
-
- def _on_selection_changed(self, new_selection, _old_selection):
- # block involuntary selection changes
- if self._selection_changed or self._model_reset:
- return
-
- indexes = new_selection.indexes()
- if not indexes:
- return
-
- project_name = indexes[0].data(QtCore.Qt.DisplayRole)
-
- if self.current_project == project_name:
- return
- self._selection_changed = True
- self.current_project = project_name
- self.project_changed.emit()
- self.refresh()
- self._selection_changed = False
-
- def refresh(self):
- selected_index = None
- model = self.project_model
- self._model_reset = True
- model.clear()
- self._model_reset = False
-
- selected_item = None
- for project_name in self.sync_server.sync_project_settings.\
- keys():
- if self.sync_server.is_paused() or \
- self.sync_server.is_project_paused(project_name):
- icon = self._get_icon("paused")
- else:
- icon = self._get_icon("synced")
-
- if project_name in self.sync_server.projects_processed:
- icon = self._get_icon("refresh")
-
- item = QtGui.QStandardItem(icon, project_name)
- model.appendRow(item)
-
- if self.current_project == project_name:
- selected_item = item
-
- if selected_item:
- selected_index = model.indexFromItem(selected_item)
-
- if len(self.sync_server.sync_project_settings.keys()) == 0:
- model.appendRow(QtGui.QStandardItem(lib.DUMMY_PROJECT))
-
- if not self.current_project:
- self.current_project = model.item(0).data(QtCore.Qt.DisplayRole)
-
- self.project_model = model
-
- if selected_index and \
- selected_index.isValid() and \
- not self._selection_changed:
- mode = QtCore.QItemSelectionModel.Select | \
- QtCore.QItemSelectionModel.Rows
- self.project_list.selectionModel().select(selected_index, mode)
-
- if self.current_project:
- self.local_site = self.sync_server.get_active_site(
- self.current_project)
- self.remote_site = self.sync_server.get_remote_site(
- self.current_project)
-
- def _can_edit(self):
- """Returns true if some site is user local site, eg. could edit"""
- return get_local_site_id() in (self.local_site, self.remote_site)
-
- def _get_icon(self, status):
- if not self.icons.get(status):
- resource_path = os.path.dirname(__file__)
- resource_path = os.path.join(resource_path, "..",
- "resources")
- pix_url = "{}/{}.png".format(resource_path, status)
- icon = QtGui.QIcon(pix_url)
- self.icons[status] = icon
- else:
- icon = self.icons[status]
- return icon
-
- def _on_context_menu(self, point):
- point_index = self.project_list.indexAt(point)
- if not point_index.isValid():
- return
-
- self.project_name = point_index.data(QtCore.Qt.DisplayRole)
-
- menu = QtWidgets.QMenu(self)
- actions_mapping = {}
-
- if self._can_edit():
- if self.sync_server.is_project_paused(self.project_name):
- action = QtWidgets.QAction("Unpause")
- actions_mapping[action] = self._unpause
- else:
- action = QtWidgets.QAction("Pause")
- actions_mapping[action] = self._pause
- menu.addAction(action)
-
- if self.local_site == get_local_site_id():
- action = QtWidgets.QAction("Clear local project")
- actions_mapping[action] = self._clear_project
- menu.addAction(action)
-
- if self.project_name not in self.sync_server.projects_processed:
- action = QtWidgets.QAction("Validate files on active site")
- actions_mapping[action] = self._validate_site
- menu.addAction(action)
-
- result = menu.exec_(QtGui.QCursor.pos())
- if result:
- to_run = actions_mapping[result]
- if to_run:
- to_run()
-
- def _pause(self):
- if self.project_name:
- self.sync_server.pause_project(self.project_name)
- self.project_name = None
- self.refresh()
-
- def _unpause(self):
- if self.project_name:
- self.sync_server.unpause_project(self.project_name)
- self.project_name = None
- self.refresh()
-
- def _clear_project(self):
- if self.project_name:
- self.sync_server.clear_project(self.project_name, self.local_site)
- self.project_name = None
- self.refresh()
-
- def _validate_site(self):
- if self.project_name:
- self.sync_server.create_validate_project_task(self.project_name,
- self.local_site)
- self.project_name = None
- self.refresh()
-
-
-class _SyncRepresentationWidget(QtWidgets.QWidget):
- """
- Summary dialog with list of representations that matches current
- settings 'local_site' and 'remote_site'.
- """
- active_changed = QtCore.Signal() # active index changed
- message_generated = QtCore.Signal(str)
-
- def _selection_changed(self, _new_selected, _all_selected):
- idxs = self.selection_model.selectedRows()
- self._selected_ids = set()
-
- for index in idxs:
- self._selected_ids.add(self.model.data(index, Qt.UserRole))
-
- def _set_selection(self):
- """
- Sets selection to 'self._selected_id' if exists.
-
- Keep selection during model refresh.
- """
- existing_ids = set()
- for selected_id in self._selected_ids:
- index = self.model.get_index(selected_id)
- if index and index.isValid():
- mode = QtCore.QItemSelectionModel.Select | \
- QtCore.QItemSelectionModel.Rows
- self.selection_model.select(index, mode)
- existing_ids.add(selected_id)
-
- self._selected_ids = existing_ids
-
- def _double_clicked(self, index):
- """
- Opens representation dialog with all files after doubleclick
- """
- # priority editing
- if self.model.can_edit:
- column_name = self.model.get_column(index.column())
- if column_name[0] in self.model.EDITABLE_COLUMNS:
- self.model.is_editing = True
- self.table_view.openPersistentEditor(index)
- return
-
- _id = self.model.data(index, Qt.UserRole)
- detail_window = SyncServerDetailWindow(
- self.sync_server, _id, self.model.project, parent=self)
- detail_window.exec()
-
- def _on_context_menu(self, point):
- """
- Shows menu with loader actions on Right-click.
-
- Supports multiple selects - adds all available actions, each
- action handles if it appropriate for item itself, if not it skips.
- """
- is_multi = len(self._selected_ids) > 1
- point_index = self.table_view.indexAt(point)
- if not point_index.isValid() and not is_multi:
- return
-
- if is_multi:
- index = self.model.get_index(list(self._selected_ids)[0])
- local_progress = self.model.data(index, LOCAL_PROGRESS_ROLE)
- remote_progress = self.model.data(index, REMOTE_PROGRESS_ROLE)
- status = self.model.data(index, STATUS_ROLE)
- else:
- local_progress = self.model.data(point_index, LOCAL_PROGRESS_ROLE)
- remote_progress = self.model.data(point_index,
- REMOTE_PROGRESS_ROLE)
- status = self.model.data(point_index, STATUS_ROLE)
-
-
- can_edit = self.model.can_edit
- action_kwarg_map, actions_mapping, menu = self._prepare_menu(
- local_progress, remote_progress, is_multi, can_edit, status)
-
- result = menu.exec_(QtGui.QCursor.pos())
- if result:
- to_run = actions_mapping[result]
- to_run_kwargs = action_kwarg_map.get(result, {})
- if to_run:
- to_run(**to_run_kwargs)
-
- self.model.refresh()
-
- def _prepare_menu(self, local_progress, remote_progress,
- is_multi, can_edit, status=None):
- menu = QtWidgets.QMenu(self)
-
- actions_mapping = {}
- action_kwarg_map = {}
-
- active_site = self.model.active_site
- remote_site = self.model.remote_site
-
- for site, progress in {active_site: local_progress,
- remote_site: remote_progress}.items():
- provider = self.sync_server.get_provider_for_site(site=site)
- if provider == 'local_drive':
- if 'studio' in site:
- txt = " studio version"
- else:
- txt = " local version"
- action = QtWidgets.QAction("Open in explorer" + txt)
- if progress == 1.0 or is_multi:
- actions_mapping[action] = self._open_in_explorer
- action_kwarg_map[action] = \
- self._get_action_kwargs(site)
- menu.addAction(action)
-
- if can_edit and (remote_progress == 1.0 or is_multi):
- action = QtWidgets.QAction("Re-sync Active site")
- action_kwarg_map[action] = self._get_action_kwargs(active_site)
- actions_mapping[action] = self._reset_site
- menu.addAction(action)
-
- if can_edit and (local_progress == 1.0 or is_multi):
- action = QtWidgets.QAction("Re-sync Remote site")
- action_kwarg_map[action] = self._get_action_kwargs(remote_site)
- actions_mapping[action] = self._reset_site
- menu.addAction(action)
-
- if can_edit and active_site == get_local_site_id():
- action = QtWidgets.QAction("Completely remove from local")
- action_kwarg_map[action] = self._get_action_kwargs(active_site)
- actions_mapping[action] = self._remove_site
- menu.addAction(action)
-
- if can_edit:
- action = QtWidgets.QAction("Change priority")
- action_kwarg_map[action] = self._get_action_kwargs(active_site)
- actions_mapping[action] = self._change_priority
- menu.addAction(action)
-
- if not actions_mapping:
- action = QtWidgets.QAction("< No action >")
- actions_mapping[action] = None
- menu.addAction(action)
-
- return action_kwarg_map, actions_mapping, menu
-
- def _pause(self, selected_ids=None):
- log.debug("Pause {}".format(selected_ids))
- for representation_id in selected_ids:
- status = lib.get_value_from_id_by_role(self.model,
- representation_id,
- STATUS_ROLE)
- if status not in [lib.STATUS[0], lib.STATUS[1]]:
- continue
- for site_name in [self.model.active_site, self.model.remote_site]:
- check_progress = self._get_progress(self.model,
- representation_id,
- site_name)
- if check_progress < 1:
- self.sync_server.pause_representation(self.model.project,
- representation_id,
- site_name)
-
- self.message_generated.emit("Paused {}".format(representation_id))
-
- def _unpause(self, selected_ids=None):
- log.debug("UnPause {}".format(selected_ids))
- for representation_id in selected_ids:
- status = lib.get_value_from_id_by_role(self.model,
- representation_id,
- STATUS_ROLE)
- if status not in lib.STATUS[3]:
- continue
- for site_name in [self.model.active_site, self.model.remote_site]:
- check_progress = self._get_progress(self.model,
- representation_id,
- site_name)
- if check_progress < 1:
- self.sync_server.unpause_representation(
- self.model.project,
- representation_id,
- site_name)
-
- self.message_generated.emit("Unpause {}".format(representation_id))
-
- # temporary here for testing, will be removed TODO
- def _add_site(self, selected_ids=None, site_name=None):
- log.debug("Add site {}:{}".format(selected_ids, site_name))
- for representation_id in selected_ids:
- item_local_site = lib.get_value_from_id_by_role(
- self.model, representation_id, LOCAL_SITE_NAME_ROLE)
- item_remote_site = lib.get_value_from_id_by_role(
- self.model, representation_id, REMOTE_SITE_NAME_ROLE)
- if site_name in [item_local_site, item_remote_site]:
- # site already exists skip
- continue
-
- try:
- self.sync_server.add_site(
- self.model.project,
- representation_id,
- site_name)
- self.message_generated.emit(
- "Site {} added for {}".format(site_name,
- representation_id))
- except ValueError as exp:
- self.message_generated.emit("Error {}".format(str(exp)))
- self.sync_server.reset_timer()
-
- def _remove_site(self, selected_ids=None, site_name=None):
- """
- Removes site record AND files.
-
- This is ONLY for representations stored on local site, which
- cannot be same as SyncServer.DEFAULT_SITE.
-
- This could only happen when artist work on local machine, not
- connected to studio mounted drives.
- """
- log.debug("Remove site {}:{}".format(selected_ids, site_name))
- for representation_id in selected_ids:
- log.info("Removing {}".format(representation_id))
- try:
- self.sync_server.remove_site(
- self.model.project,
- representation_id,
- site_name,
- True)
- self.message_generated.emit(
- "Site {} removed".format(site_name))
- except ValueError as exp:
- self.message_generated.emit("Error {}".format(str(exp)))
-
- self.model.refresh(
- load_records=self.model._rec_loaded)
- self.sync_server.reset_timer()
-
- def _reset_site(self, selected_ids=None, site_name=None):
- """
- Removes errors or success metadata for particular file >> forces
- redo of upload/download
- """
- log.debug("Reset site {}:{}".format(selected_ids, site_name))
- for representation_id in selected_ids:
- check_progress = self._get_progress(self.model, representation_id,
- site_name, True)
-
- # do not reset if opposite side is not fully there
- if check_progress != 1:
- log.debug("Not fully available {} on other side, skipping".
- format(check_progress))
- continue
-
- self.sync_server.reset_site_on_representation(
- self.model.project,
- representation_id,
- site_name=site_name,
- force=True)
-
- self.model.refresh(
- load_records=self.model._rec_loaded)
- self.sync_server.reset_timer()
-
- def _open_in_explorer(self, selected_ids=None, site_name=None):
- log.debug("Open in Explorer {}:{}".format(selected_ids, site_name))
- for selected_id in selected_ids:
- fpath = lib.get_value_from_id_by_role(self.model, selected_id,
- PATH_ROLE)
- project = self.model.project
- fpath = self.sync_server.get_local_file_path(project,
- site_name,
- fpath)
-
- fpath = os.path.normpath(os.path.dirname(fpath))
- if os.path.isdir(fpath):
- if 'win' in sys.platform: # windows
- subprocess.Popen('explorer "%s"' % fpath)
- elif sys.platform == 'darwin': # macOS
- subprocess.Popen(['open', fpath])
- else: # linux
- try:
- subprocess.Popen(['xdg-open', fpath])
- except OSError:
- raise OSError('unsupported xdg-open call??')
-
- def _change_priority(self, **kwargs):
- """Open editor to change priority on first selected row"""
- if self._selected_ids:
- # get_index returns dummy index with column equals to 0
- index = self.model.get_index(list(self._selected_ids)[0])
- column_no = self.model.get_header_index("priority") # real column
- real_index = self.model.index(index.row(), column_no)
- self.model.is_editing = True
- self.table_view.openPersistentEditor(real_index)
-
- def _get_progress(self, model, representation_id,
- site_name, opposite=False):
- """Returns progress value according to site (side)"""
- local_progress = lib.get_value_from_id_by_role(model,
- representation_id,
- LOCAL_PROGRESS_ROLE)
- remote_progress = lib.get_value_from_id_by_role(model,
- representation_id,
- REMOTE_PROGRESS_ROLE)
- progress = {'local': local_progress,
- 'remote': remote_progress}
- side = 'remote'
- if site_name == self.model.active_site:
- side = 'local'
- if opposite:
- side = 'remote' if side == 'local' else 'local'
-
- return progress[side]
-
- def _get_action_kwargs(self, site_name):
- """Default format of kwargs for action"""
- return {"selected_ids": self._selected_ids, "site_name": site_name}
-
- def _save_scrollbar(self):
- self._scrollbar_pos = self.table_view.verticalScrollBar().value()
-
- def _set_scrollbar(self):
- if self._scrollbar_pos:
- self.table_view.verticalScrollBar().setValue(self._scrollbar_pos)
-
-
-class SyncRepresentationSummaryWidget(_SyncRepresentationWidget):
-
- default_widths = (
- ("asset", 190),
- ("subset", 170),
- ("version", 60),
- ("representation", 145),
- ("local_site", 160),
- ("remote_site", 160),
- ("files_count", 50),
- ("files_size", 60),
- ("priority", 70),
- ("status", 110)
- )
-
- def __init__(self, sync_server, project=None, parent=None):
- super(SyncRepresentationSummaryWidget, self).__init__(parent)
-
- self.sync_server = sync_server
-
- self._selected_ids = set() # keep last selected _id
-
- txt_filter = QtWidgets.QLineEdit()
- txt_filter.setPlaceholderText("Quick filter representations..")
- txt_filter.setClearButtonEnabled(True)
- txt_filter.addAction(
- qtawesome.icon("fa.filter", color="gray"),
- QtWidgets.QLineEdit.LeadingPosition)
- self.txt_filter = txt_filter
-
- self._scrollbar_pos = None
-
- top_bar_layout = QtWidgets.QHBoxLayout()
- top_bar_layout.addWidget(self.txt_filter)
-
- table_view = QtWidgets.QTableView()
- headers = [item[0] for item in self.default_widths]
-
- model = SyncRepresentationSummaryModel(sync_server, headers, project,
- parent=self)
- table_view.setModel(model)
- table_view.setContextMenuPolicy(QtCore.Qt.CustomContextMenu)
- table_view.setSelectionMode(
- QtWidgets.QAbstractItemView.ExtendedSelection)
- table_view.setSelectionBehavior(
- QtWidgets.QAbstractItemView.SelectRows)
- table_view.horizontalHeader().setSortIndicator(
- -1, Qt.AscendingOrder)
- table_view.setAlternatingRowColors(True)
- table_view.verticalHeader().hide()
- table_view.viewport().setAttribute(QtCore.Qt.WA_Hover, True)
-
- column = table_view.model().get_header_index("local_site")
- delegate = delegates.ImageDelegate(self, side="local")
- table_view.setItemDelegateForColumn(column, delegate)
-
- column = table_view.model().get_header_index("remote_site")
- delegate = delegates.ImageDelegate(self, side="remote")
- table_view.setItemDelegateForColumn(column, delegate)
-
- column = table_view.model().get_header_index("priority")
- priority_delegate = delegates.PriorityDelegate(self)
- table_view.setItemDelegateForColumn(column, priority_delegate)
-
- layout = QtWidgets.QVBoxLayout(self)
- layout.setContentsMargins(0, 0, 0, 0)
- layout.addLayout(top_bar_layout)
- layout.addWidget(table_view)
-
- self.table_view = table_view
- self.model = model
-
- horizontal_header = HorizontalHeader(self)
-
- table_view.setHorizontalHeader(horizontal_header)
- table_view.setSortingEnabled(True)
-
- for column_name, width in self.default_widths:
- idx = model.get_header_index(column_name)
- table_view.setColumnWidth(idx, width)
-
- table_view.doubleClicked.connect(self._double_clicked)
- self.txt_filter.textChanged.connect(lambda: model.set_word_filter(
- self.txt_filter.text()))
- table_view.customContextMenuRequested.connect(self._on_context_menu)
-
- model.refresh_started.connect(self._save_scrollbar)
- model.refresh_finished.connect(self._set_scrollbar)
- model.modelReset.connect(self._set_selection)
-
- self.selection_model = self.table_view.selectionModel()
- self.selection_model.selectionChanged.connect(self._selection_changed)
-
- def _prepare_menu(self, local_progress, remote_progress,
- is_multi, can_edit, status=None):
- action_kwarg_map, actions_mapping, menu = \
- super()._prepare_menu(local_progress, remote_progress,
- is_multi, can_edit)
-
- if can_edit and (
- status in [lib.STATUS[0], lib.STATUS[1]] or is_multi):
- action = QtWidgets.QAction("Pause in queue")
- actions_mapping[action] = self._pause
- # pause handles which site_name it will pause itself
- action_kwarg_map[action] = {"selected_ids": self._selected_ids}
- menu.addAction(action)
-
- if can_edit and (status == lib.STATUS[3] or is_multi):
- action = QtWidgets.QAction("Unpause in queue")
- actions_mapping[action] = self._unpause
- action_kwarg_map[action] = {"selected_ids": self._selected_ids}
- menu.addAction(action)
-
- return action_kwarg_map, actions_mapping, menu
-
-
-class SyncServerDetailWindow(QtWidgets.QDialog):
- """Wrapper window for SyncRepresentationDetailWidget
-
- Creates standalone window with list of files for selected repre_id.
- """
- def __init__(self, sync_server, _id, project, parent=None):
- log.debug(
- "!!! SyncServerDetailWindow _id:: {}".format(_id))
- super(SyncServerDetailWindow, self).__init__(parent)
- self.setWindowFlags(QtCore.Qt.Window)
- self.setFocusPolicy(QtCore.Qt.StrongFocus)
-
- self.setStyleSheet(style.load_stylesheet())
- self.setWindowIcon(QtGui.QIcon(style.app_icon_path()))
- self.resize(1000, 400)
-
- body = QtWidgets.QWidget()
- footer = QtWidgets.QWidget()
- footer.setFixedHeight(20)
-
- container = SyncRepresentationDetailWidget(sync_server, _id, project,
- parent=self)
- body_layout = QtWidgets.QHBoxLayout(body)
- body_layout.addWidget(container)
- body_layout.setContentsMargins(0, 0, 0, 0)
-
- self.message = QtWidgets.QLabel()
- self.message.hide()
-
- footer_layout = QtWidgets.QVBoxLayout(footer)
- footer_layout.addWidget(self.message)
- footer_layout.setContentsMargins(0, 0, 0, 0)
-
- layout = QtWidgets.QVBoxLayout(self)
- layout.addWidget(body)
- layout.addWidget(footer)
-
- self.setWindowTitle("Sync Representation Detail")
-
-
-class SyncRepresentationDetailWidget(_SyncRepresentationWidget):
- """
- Widget to display list of synchronizable files for single repre.
-
- Args:
- _id (str): representation _id
- project (str): name of project with repre
- parent (QDialog): SyncServerDetailWindow
- """
- active_changed = QtCore.Signal() # active index changed
-
- default_widths = (
- ("file", 290),
- ("local_site", 185),
- ("remote_site", 185),
- ("size", 60),
- ("priority", 60),
- ("status", 110)
- )
-
- def __init__(self, sync_server, _id=None, project=None, parent=None):
- super(SyncRepresentationDetailWidget, self).__init__(parent)
-
- log.debug("Representation_id:{}".format(_id))
- self.project = project
-
- self.sync_server = sync_server
-
- self.representation_id = _id
- self._selected_ids = set()
-
- self.txt_filter = QtWidgets.QLineEdit()
- self.txt_filter.setPlaceholderText("Quick filter representation..")
- self.txt_filter.setClearButtonEnabled(True)
- self.txt_filter.addAction(qtawesome.icon("fa.filter", color="gray"),
- QtWidgets.QLineEdit.LeadingPosition)
-
- self._scrollbar_pos = None
-
- top_bar_layout = QtWidgets.QHBoxLayout()
- top_bar_layout.addWidget(self.txt_filter)
-
- table_view = QtWidgets.QTableView()
- headers = [item[0] for item in self.default_widths]
-
- model = SyncRepresentationDetailModel(sync_server, headers, _id,
- project)
- model.is_running = True
-
- table_view.setModel(model)
- table_view.setContextMenuPolicy(QtCore.Qt.CustomContextMenu)
- table_view.setSelectionMode(
- QtWidgets.QAbstractItemView.ExtendedSelection)
- table_view.setSelectionBehavior(
- QtWidgets.QTableView.SelectRows)
- table_view.horizontalHeader().setSortIndicator(-1, Qt.AscendingOrder)
- table_view.horizontalHeader().setSortIndicatorShown(True)
- table_view.setAlternatingRowColors(True)
- table_view.verticalHeader().hide()
-
- column = model.get_header_index("local_site")
- delegate = delegates.ImageDelegate(self, side="local")
- table_view.setItemDelegateForColumn(column, delegate)
-
- column = model.get_header_index("remote_site")
- delegate = delegates.ImageDelegate(self, side="remote")
- table_view.setItemDelegateForColumn(column, delegate)
-
- if model.can_edit:
- column = table_view.model().get_header_index("priority")
- priority_delegate = delegates.PriorityDelegate(self)
- table_view.setItemDelegateForColumn(column, priority_delegate)
-
- layout = QtWidgets.QVBoxLayout(self)
- layout.setContentsMargins(0, 0, 0, 0)
- layout.addLayout(top_bar_layout)
- layout.addWidget(table_view)
-
- self.model = model
-
- self.selection_model = table_view.selectionModel()
- self.selection_model.selectionChanged.connect(self._selection_changed)
-
- horizontal_header = HorizontalHeader(self)
-
- table_view.setHorizontalHeader(horizontal_header)
- table_view.setSortingEnabled(True)
-
- for column_name, width in self.default_widths:
- idx = model.get_header_index(column_name)
- table_view.setColumnWidth(idx, width)
-
- self.table_view = table_view
-
- self.txt_filter.textChanged.connect(lambda: model.set_word_filter(
- self.txt_filter.text()))
- table_view.doubleClicked.connect(self._double_clicked)
- table_view.customContextMenuRequested.connect(self._on_context_menu)
-
- model.refresh_started.connect(self._save_scrollbar)
- model.refresh_finished.connect(self._set_scrollbar)
- model.modelReset.connect(self._set_selection)
-
- def _double_clicked(self, index):
- """
- Opens representation dialog with all files after doubleclick
- """
- # priority editing
- if self.model.can_edit:
- column_name = self.model.get_column(index.column())
- if column_name[0] in self.model.EDITABLE_COLUMNS:
- self.model.is_editing = True
- self.table_view.openPersistentEditor(index)
- return
-
- def _show_detail(self, selected_ids=None):
- """
- Shows windows with error message for failed sync of a file.
- """
- detail_window = SyncRepresentationErrorWindow(self.model, selected_ids)
-
- detail_window.exec()
-
- def _prepare_menu(self, local_progress, remote_progress,
- is_multi, can_edit, status=None):
- """Adds view (and model) dependent actions to default ones"""
- action_kwarg_map, actions_mapping, menu = \
- super()._prepare_menu(local_progress, remote_progress,
- is_multi, can_edit, status)
-
- if status == lib.STATUS[2] or is_multi:
- action = QtWidgets.QAction("Open error detail")
- actions_mapping[action] = self._show_detail
- action_kwarg_map[action] = {"selected_ids": self._selected_ids}
-
- menu.addAction(action)
-
- return action_kwarg_map, actions_mapping, menu
-
- def _reset_site(self, selected_ids=None, site_name=None):
- """
- Removes errors or success metadata for particular file >> forces
- redo of upload/download
- """
- for file_id in selected_ids:
- check_progress = self._get_progress(self.model, file_id,
- site_name, True)
-
- # do not reset if opposite side is not fully there
- if check_progress != 1:
- log.debug("Not fully available {} on other side, skipping".
- format(check_progress))
- continue
-
- self.sync_server.reset_site_on_representation(
- self.model.project,
- self.representation_id,
- site_name=site_name,
- file_id=file_id,
- force=True)
- self.model.refresh(
- load_records=self.model._rec_loaded)
-
-
-class SyncRepresentationErrorWindow(QtWidgets.QDialog):
- """Wrapper window to show errors during sync on file(s)"""
- def __init__(self, model, selected_ids, parent=None):
- super(SyncRepresentationErrorWindow, self).__init__(parent)
- self.setWindowFlags(QtCore.Qt.Window)
- self.setFocusPolicy(QtCore.Qt.StrongFocus)
-
- self.setStyleSheet(style.load_stylesheet())
- self.setWindowIcon(QtGui.QIcon(style.app_icon_path()))
- self.resize(900, 150)
-
- body = QtWidgets.QWidget()
-
- container = SyncRepresentationErrorWidget(model,
- selected_ids,
- parent=self)
- body_layout = QtWidgets.QHBoxLayout(body)
- body_layout.addWidget(container)
- body_layout.setContentsMargins(0, 0, 0, 0)
-
- message = QtWidgets.QLabel()
- message.hide()
-
- layout = QtWidgets.QVBoxLayout(self)
- layout.addWidget(body)
-
- self.setLayout(body_layout)
- self.setWindowTitle("Sync Representation Error Detail")
-
-
-class SyncRepresentationErrorWidget(QtWidgets.QWidget):
- """
- Dialog to show when sync error happened, prints formatted error message
- """
- def __init__(self, model, selected_ids, parent=None):
- super(SyncRepresentationErrorWidget, self).__init__(parent)
-
- layout = QtWidgets.QVBoxLayout(self)
-
- no_errors = True
- for file_id in selected_ids:
- created_dt = lib.get_value_from_id_by_role(model, file_id,
- LOCAL_DATE_ROLE)
- sync_dt = lib.get_value_from_id_by_role(model, file_id,
- REMOTE_DATE_ROLE)
- errors = lib.get_value_from_id_by_role(model, file_id,
- ERROR_ROLE)
- if not created_dt or not sync_dt or not errors:
- continue
-
- tries = lib.get_value_from_id_by_role(model, file_id,
- TRIES_ROLE)
-
- no_errors = False
- dt = max(created_dt, sync_dt)
-
- txts = []
- txts.append("{}: {}Last update date ",
- pretty_timestamp(dt)))
- txts.append("{}: {}Retries ",
- str(tries)))
- txts.append("{}: {}Error message ",
- errors))
-
- text_area = QtWidgets.QTextEdit("\n\n".join(txts))
- text_area.setReadOnly(True)
- layout.addWidget(text_area)
-
- if no_errors:
- text_area = QtWidgets.QTextEdit()
- text_area.setText("No errors located ")
- text_area.setReadOnly(True)
- layout.addWidget(text_area)
-
-
-class HorizontalHeader(QtWidgets.QHeaderView):
- """Reiplemented QHeaderView to contain clickable changeable button"""
- def __init__(self, parent=None):
- super(HorizontalHeader, self).__init__(QtCore.Qt.Horizontal, parent)
- self._parent = parent
- self.checked_values = {}
-
- self.setModel(self._parent.model)
-
- self.setSectionsClickable(True)
-
- self.menu_items_dict = {}
- self.menu = None
- self.header_cells = []
- self.filter_buttons = {}
-
- self.filter_icon = qtawesome.icon("fa.filter", color="gray")
- self.filter_set_icon = qtawesome.icon("fa.filter", color="white")
-
- self.init_layout()
-
- self._resetting = False
-
- @property
- def model(self):
- """Keep model synchronized with parent widget"""
- return self._parent.model
-
- def init_layout(self):
- """Initial preparation of header's content"""
- for column_idx in range(self.model.columnCount()):
- column_name, column_label = self.model.get_column(column_idx)
- filter_rec = self.model.get_filters().get(column_name)
- if not filter_rec:
- continue
-
- icon = self.filter_icon
- button = QtWidgets.QPushButton(icon, "", self)
-
- button.setFixedSize(24, 24)
- button.setStyleSheet(
- "QPushButton::menu-indicator{width:0px;}"
- "QPushButton{border: none;background: transparent;}")
- button.clicked.connect(partial(self._get_menu,
- column_name, column_idx))
- button.setFlat(True)
- self.filter_buttons[column_name] = button
-
- def showEvent(self, event):
- """Paint header"""
- super(HorizontalHeader, self).showEvent(event)
-
- for i in range(len(self.header_cells)):
- cell_content = self.header_cells[i]
- cell_content.setGeometry(self.sectionViewportPosition(i), 0,
- self.sectionSize(i) - 1, self.height())
-
- cell_content.show()
-
- def _set_filter_icon(self, column_name):
- """Set different states of button depending on its engagement"""
- button = self.filter_buttons.get(column_name)
- if button:
- if self.checked_values.get(column_name):
- button.setIcon(self.filter_set_icon)
- else:
- button.setIcon(self.filter_icon)
-
- def _reset_filter(self, column_name):
- """
- Remove whole column from filter >> not in $match at all (faster)
- """
- self._resetting = True # mark changes to consume them
- if self.checked_values.get(column_name) is not None:
- self.checked_values.pop(column_name)
- self._set_filter_icon(column_name)
- self._filter_and_refresh_model_and_menu(column_name, True, True)
- self._resetting = False
-
- def _apply_filter(self, column_name, values, state):
- """
- Sets 'values' to specific 'state' (checked/unchecked),
- sends to model.
- """
- if self._resetting: # event triggered by _resetting, skip it
- return
-
- self._update_checked_values(column_name, values, state)
- self._set_filter_icon(column_name)
- self._filter_and_refresh_model_and_menu(column_name, True, False)
-
- def _apply_text_filter(self, column_name, items, line_edit):
- """
- Resets all checkboxes, prefers inserted text.
- """
- le_text = line_edit.text()
- self._update_checked_values(column_name, items, 0) # reset other
- if self.checked_values.get(column_name) is not None or \
- le_text == '':
- self.checked_values.pop(column_name) # reset during typing
-
- if le_text:
- self._update_checked_values(column_name, {le_text: le_text}, 2)
- self._set_filter_icon(column_name)
- self._filter_and_refresh_model_and_menu(column_name, True, True)
-
- def _filter_and_refresh_model_and_menu(self, column_name,
- model=True, menu=True):
- """
- Refresh model and its content and possibly menu for big changes.
- """
- if model:
- self.model.set_column_filtering(self.checked_values)
- self.model.refresh()
- if menu:
- self._menu_refresh(column_name)
-
- def _get_menu(self, column_name, index):
- """Prepares content of menu for 'column_name'"""
- menu = QtWidgets.QMenu(self)
- filter_rec = self.model.get_filters()[column_name]
- self.menu_items_dict[column_name] = filter_rec.values()
-
- # text filtering only if labels same as values, not if codes are used
- if 'text' in filter_rec.search_variants():
- line_edit = QtWidgets.QLineEdit(menu)
- line_edit.setClearButtonEnabled(True)
- line_edit.addAction(self.filter_icon,
- QtWidgets.QLineEdit.LeadingPosition)
-
- line_edit.setFixedHeight(line_edit.height())
- txt = ""
- if self.checked_values.get(column_name):
- txt = list(self.checked_values.get(column_name).keys())[0]
- line_edit.setText(txt)
-
- action_le = QtWidgets.QWidgetAction(menu)
- action_le.setDefaultWidget(line_edit)
- line_edit.textChanged.connect(
- partial(self._apply_text_filter, column_name,
- filter_rec.values(), line_edit))
- menu.addAction(action_le)
- menu.addSeparator()
-
- if 'checkbox' in filter_rec.search_variants():
- action_all = QtWidgets.QAction("All", self)
- action_all.triggered.connect(partial(self._reset_filter,
- column_name))
- menu.addAction(action_all)
-
- action_none = QtWidgets.QAction("Unselect all", self)
- state_unchecked = 0
- action_none.triggered.connect(partial(self._apply_filter,
- column_name,
- filter_rec.values(),
- state_unchecked))
- menu.addAction(action_none)
- menu.addSeparator()
-
- # nothing explicitly >> ALL implicitly >> first time
- if self.checked_values.get(column_name) is None:
- checked_keys = self.menu_items_dict[column_name].keys()
- else:
- checked_keys = self.checked_values[column_name]
-
- for value, label in self.menu_items_dict[column_name].items():
- checkbox = QtWidgets.QCheckBox(str(label), menu)
-
- # temp
- checkbox.setStyleSheet("QCheckBox{spacing: 5px;"
- "padding:5px 5px 5px 5px;}")
- if value in checked_keys:
- checkbox.setChecked(True)
-
- action = QtWidgets.QWidgetAction(menu)
- action.setDefaultWidget(checkbox)
-
- checkbox.stateChanged.connect(partial(self._apply_filter,
- column_name, {value: label}))
- menu.addAction(action)
-
- self.menu = menu
-
- self._show_menu(index, menu)
-
- def _show_menu(self, index, menu):
- """Shows 'menu' under header column of 'index'"""
- global_pos_point = self.mapToGlobal(
- QtCore.QPoint(self.sectionViewportPosition(index), 0))
- menu.setMinimumWidth(self.sectionSize(index))
- menu.setMinimumHeight(self.height())
- menu.exec_(QtCore.QPoint(global_pos_point.x(),
- global_pos_point.y() + self.height()))
-
- def _menu_refresh(self, column_name):
- """
- Reset boxes after big change - word filtering or reset
- """
- for action in self.menu.actions():
- if not isinstance(action, QtWidgets.QWidgetAction):
- continue
-
- widget = action.defaultWidget()
- if not isinstance(widget, QtWidgets.QCheckBox):
- continue
-
- if not self.checked_values.get(column_name) or \
- widget.text() in self.checked_values[column_name].values():
- widget.setChecked(True)
- else:
- widget.setChecked(False)
-
- def _update_checked_values(self, column_name, values, state):
- """
- Modify dictionary of set values in columns for filtering.
-
- Modifies 'self.checked_values'
- """
- copy_menu_items = dict(self.menu_items_dict[column_name])
- checked = self.checked_values.get(column_name, copy_menu_items)
- set_items = dict(values.items()) # prevent dict change during loop
- for value, label in set_items.items():
- if state == 2 and label: # checked
- checked[value] = label
- elif state == 0 and checked.get(value):
- checked.pop(value)
-
- self.checked_values[column_name] = checked
-
- def paintEvent(self, event):
- self._fix_size()
- super(HorizontalHeader, self).paintEvent(event)
-
- def _fix_size(self):
- for column_idx in range(self.model.columnCount()):
- vis_index = self.visualIndex(column_idx)
- index = self.logicalIndex(vis_index)
- section_width = self.sectionSize(index)
-
- column_name = self.model.headerData(column_idx,
- QtCore.Qt.Horizontal,
- HEADER_NAME_ROLE)
- button = self.filter_buttons.get(column_name)
- if not button:
- continue
-
- pos_x = self.sectionViewportPosition(
- index) + section_width - self.height()
-
- pos_y = 0
- if button.height() < self.height():
- pos_y = int((self.height() - button.height()) / 2)
- button.setGeometry(
- pos_x,
- pos_y,
- self.height(),
- self.height())
diff --git a/openpype/modules/default_modules/sync_server/utils.py b/openpype/modules/default_modules/sync_server/utils.py
deleted file mode 100644
index 85e4e03f777..00000000000
--- a/openpype/modules/default_modules/sync_server/utils.py
+++ /dev/null
@@ -1,40 +0,0 @@
-import time
-from openpype.api import Logger
-log = Logger().get_logger("SyncServer")
-
-
-class ResumableError(Exception):
- """Error which could be temporary, skip current loop, try next time"""
- pass
-
-
-class SyncStatus:
- DO_NOTHING = 0
- DO_UPLOAD = 1
- DO_DOWNLOAD = 2
-
-
-def time_function(method):
- """ Decorator to print how much time function took.
- For debugging.
- Depends on presence of 'log' object
- """
-
- def timed(*args, **kw):
- ts = time.time()
- result = method(*args, **kw)
- te = time.time()
- if 'log_time' in kw:
- name = kw.get('log_name', method.__name__.upper())
- kw['log_time'][name] = int((te - ts) * 1000)
- else:
- log.debug('%r %2.2f ms' % (method.__name__, (te - ts) * 1000))
- return result
-
- return timed
-
-
-class EditableScopes:
- SYSTEM = 0
- PROJECT = 1
- LOCAL = 2
diff --git a/openpype/modules/ftrack/__init__.py b/openpype/modules/ftrack/__init__.py
new file mode 100644
index 00000000000..e520f08337e
--- /dev/null
+++ b/openpype/modules/ftrack/__init__.py
@@ -0,0 +1,13 @@
+from .ftrack_module import (
+ FtrackModule,
+ FTRACK_MODULE_DIR,
+
+ resolve_ftrack_url,
+)
+
+__all__ = (
+ "FtrackModule",
+ "FTRACK_MODULE_DIR",
+
+ "resolve_ftrack_url",
+)
diff --git a/openpype/modules/default_modules/ftrack/event_handlers_server/action_clone_review_session.py b/openpype/modules/ftrack/event_handlers_server/action_clone_review_session.py
similarity index 100%
rename from openpype/modules/default_modules/ftrack/event_handlers_server/action_clone_review_session.py
rename to openpype/modules/ftrack/event_handlers_server/action_clone_review_session.py
diff --git a/openpype/modules/ftrack/event_handlers_server/action_create_review_session.py b/openpype/modules/ftrack/event_handlers_server/action_create_review_session.py
new file mode 100644
index 00000000000..21382007a0f
--- /dev/null
+++ b/openpype/modules/ftrack/event_handlers_server/action_create_review_session.py
@@ -0,0 +1,311 @@
+import threading
+import datetime
+import copy
+import collections
+
+import ftrack_api
+
+from openpype.lib import get_datetime_data
+from openpype.settings.lib import (
+ get_project_settings,
+ get_default_project_settings
+)
+from openpype_modules.ftrack.lib import ServerAction
+
+
+class CreateDailyReviewSessionServerAction(ServerAction):
+ """Create daily review session object per project.
+
+ Action creates review sessions based on settings. Settings define if is
+ action enabled and what is a template for review session name. Logic works
+ in a way that if review session with the name already exists then skip
+ process. If review session for current day does not exist but yesterdays
+ review exists and is empty then yesterdays is renamed otherwise creates
+ new review session.
+
+ Also contains cycle creation of dailies which is triggered each morning.
+ This option must be enabled in project settings. Cycle creation is also
+ checked on registration of action.
+ """
+
+ identifier = "create.daily.review.session"
+ #: Action label.
+ label = "OpenPype Admin"
+ variant = "- Create Daily Review Session (Server)"
+ #: Action description.
+ description = "Manually create daily review session"
+ role_list = {"Pypeclub", "Administrator", "Project Manager"}
+
+ settings_key = "create_daily_review_session"
+ default_template = "{yy}{mm}{dd}"
+
+ def __init__(self, *args, **kwargs):
+ super(CreateDailyReviewSessionServerAction, self).__init__(
+ *args, **kwargs
+ )
+
+ self._cycle_timer = None
+ self._last_cyle_time = None
+ self._day_delta = datetime.timedelta(days=1)
+
+ def discover(self, session, entities, event):
+ """Show action only on AssetVersions."""
+
+ valid_selection = False
+ for ent in event["data"]["selection"]:
+ # Ignore entities that are not tasks or projects
+ if ent["entityType"].lower() in (
+ "show", "task", "reviewsession", "assetversion"
+ ):
+ valid_selection = True
+ break
+
+ if not valid_selection:
+ return False
+ return self.valid_roles(session, entities, event)
+
+ def launch(self, session, entities, event):
+ project_entity = self.get_project_from_entity(entities[0], session)
+ project_name = project_entity["full_name"]
+ project_settings = self.get_project_settings_from_event(
+ event, project_name
+ )
+ action_settings = self._extract_action_settings(project_settings)
+ project_name_by_id = {
+ project_entity["id"]: project_name
+ }
+ settings_by_project_id = {
+ project_entity["id"]: action_settings
+ }
+ self._process_review_session(
+ session, settings_by_project_id, project_name_by_id
+ )
+ return True
+
+ def _calculate_next_cycle_delta(self):
+ studio_default_settings = get_default_project_settings()
+ action_settings = (
+ studio_default_settings
+ ["ftrack"]
+ [self.settings_frack_subkey]
+ [self.settings_key]
+ )
+ cycle_hour_start = action_settings.get("cycle_hour_start")
+ if not cycle_hour_start:
+ h = m = s = 0
+ else:
+ h, m, s = cycle_hour_start
+
+ # Create threading timer which will trigger creation of report
+ # at the 00:00:01 of next day
+ # - callback will trigger another timer which will have 1 day offset
+ now = datetime.datetime.now()
+ # Create object of today morning
+ expected_next_trigger = datetime.datetime(
+ now.year, now.month, now.day, h, m, s
+ )
+ if expected_next_trigger > now:
+ seconds = (expected_next_trigger - now).total_seconds()
+ else:
+ expected_next_trigger += self._day_delta
+ seconds = (expected_next_trigger - now).total_seconds()
+ return seconds, expected_next_trigger
+
+ def register(self, *args, **kwargs):
+ """Override register to be able trigger """
+ # Register server action as would be normally
+ super(CreateDailyReviewSessionServerAction, self).register(
+ *args, **kwargs
+ )
+
+ seconds_delta, cycle_time = self._calculate_next_cycle_delta()
+
+ # Store cycle time which will be used to create next timer
+ self._last_cyle_time = cycle_time
+ # Create timer thread
+ self._cycle_timer = threading.Timer(
+ seconds_delta, self._timer_callback
+ )
+ self._cycle_timer.start()
+
+ self._check_review_session()
+
+ def _timer_callback(self):
+ if (
+ self._cycle_timer is not None
+ and self._last_cyle_time is not None
+ ):
+ seconds_delta, cycle_time = self._calculate_next_cycle_delta()
+ self._last_cyle_time = cycle_time
+
+ self._cycle_timer = threading.Timer(
+ seconds_delta, self._timer_callback
+ )
+ self._cycle_timer.start()
+ self._check_review_session()
+
+ def _check_review_session(self):
+ session = ftrack_api.Session(
+ server_url=self.session.server_url,
+ api_key=self.session.api_key,
+ api_user=self.session.api_user,
+ auto_connect_event_hub=False
+ )
+ project_entities = session.query(
+ "select id, full_name from Project"
+ ).all()
+ project_names_by_id = {
+ project_entity["id"]: project_entity["full_name"]
+ for project_entity in project_entities
+ }
+
+ action_settings_by_project_id = self._get_action_settings(
+ project_names_by_id
+ )
+ enabled_action_settings_by_project_id = {}
+ for item in action_settings_by_project_id.items():
+ project_id, action_settings = item
+ if action_settings.get("cycle_enabled"):
+ enabled_action_settings_by_project_id[project_id] = (
+ action_settings
+ )
+
+ if not enabled_action_settings_by_project_id:
+ self.log.info((
+ "There are no projects that have enabled"
+ " cycle review sesison creation"
+ ))
+
+ else:
+ self._process_review_session(
+ session,
+ enabled_action_settings_by_project_id,
+ project_names_by_id
+ )
+
+ session.close()
+
+ def _process_review_session(
+ self, session, settings_by_project_id, project_names_by_id
+ ):
+ review_sessions = session.query((
+ "select id, name, project_id"
+ " from ReviewSession where project_id in ({})"
+ ).format(self.join_query_keys(settings_by_project_id))).all()
+
+ review_sessions_by_project_id = collections.defaultdict(list)
+ for review_session in review_sessions:
+ project_id = review_session["project_id"]
+ review_sessions_by_project_id[project_id].append(review_session)
+
+ # Prepare fill data for today's review sesison and yesterdays
+ now = datetime.datetime.now()
+ today_obj = datetime.datetime(
+ now.year, now.month, now.day, 0, 0, 0
+ )
+ yesterday_obj = today_obj - self._day_delta
+
+ today_fill_data = get_datetime_data(today_obj)
+ yesterday_fill_data = get_datetime_data(yesterday_obj)
+
+ # Loop through projects and try to create daily reviews
+ for project_id, action_settings in settings_by_project_id.items():
+ review_session_template = (
+ action_settings["review_session_template"]
+ ).strip() or self.default_template
+
+ today_project_fill_data = copy.deepcopy(today_fill_data)
+ yesterday_project_fill_data = copy.deepcopy(yesterday_fill_data)
+ project_name = project_names_by_id[project_id]
+ today_project_fill_data["project_name"] = project_name
+ yesterday_project_fill_data["project_name"] = project_name
+
+ today_session_name = self._fill_review_template(
+ review_session_template, today_project_fill_data
+ )
+ yesterday_session_name = self._fill_review_template(
+ review_session_template, yesterday_project_fill_data
+ )
+ # Skip if today's session name could not be filled
+ if not today_session_name:
+ continue
+
+ # Find matchin review session
+ project_review_sessions = review_sessions_by_project_id[project_id]
+ todays_session = None
+ yesterdays_session = None
+ for review_session in project_review_sessions:
+ session_name = review_session["name"]
+ if session_name == today_session_name:
+ todays_session = review_session
+ break
+ elif session_name == yesterday_session_name:
+ yesterdays_session = review_session
+
+ # Skip if today's session already exist
+ if todays_session is not None:
+ self.log.debug((
+ "Todays ReviewSession \"{}\""
+ " in project \"{}\" already exists"
+ ).format(today_session_name, project_name))
+ continue
+
+ # Check if there is yesterday's session and is empty
+ # - in that case just rename it
+ if (
+ yesterdays_session is not None
+ and len(yesterdays_session["review_session_objects"]) == 0
+ ):
+ self.log.debug((
+ "Renaming yesterdays empty review session \"{}\" to \"{}\""
+ " in project \"{}\""
+ ).format(
+ yesterday_session_name, today_session_name, project_name
+ ))
+ yesterdays_session["name"] = today_session_name
+ session.commit()
+ continue
+
+ # Create new review session with new name
+ self.log.debug((
+ "Creating new review session \"{}\" in project \"{}\""
+ ).format(today_session_name, project_name))
+ session.create("ReviewSession", {
+ "project_id": project_id,
+ "name": today_session_name
+ })
+ session.commit()
+
+ def _get_action_settings(self, project_names_by_id):
+ settings_by_project_id = {}
+ for project_id, project_name in project_names_by_id.items():
+ project_settings = get_project_settings(project_name)
+ action_settings = self._extract_action_settings(project_settings)
+ settings_by_project_id[project_id] = action_settings
+ return settings_by_project_id
+
+ def _extract_action_settings(self, project_settings):
+ return (
+ project_settings
+ .get("ftrack", {})
+ .get(self.settings_frack_subkey, {})
+ .get(self.settings_key)
+ ) or {}
+
+ def _fill_review_template(self, template, data):
+ output = None
+ try:
+ output = template.format(**data)
+ except Exception:
+ self.log.warning(
+ (
+ "Failed to fill review session template {} with data {}"
+ ).format(template, data),
+ exc_info=True
+ )
+ return output
+
+
+def register(session):
+ '''Register plugin. Called when used as an plugin.'''
+ CreateDailyReviewSessionServerAction(session).register()
diff --git a/openpype/modules/default_modules/ftrack/event_handlers_server/action_multiple_notes.py b/openpype/modules/ftrack/event_handlers_server/action_multiple_notes.py
similarity index 100%
rename from openpype/modules/default_modules/ftrack/event_handlers_server/action_multiple_notes.py
rename to openpype/modules/ftrack/event_handlers_server/action_multiple_notes.py
diff --git a/openpype/modules/ftrack/event_handlers_server/action_prepare_project.py b/openpype/modules/ftrack/event_handlers_server/action_prepare_project.py
new file mode 100644
index 00000000000..332648cd028
--- /dev/null
+++ b/openpype/modules/ftrack/event_handlers_server/action_prepare_project.py
@@ -0,0 +1,415 @@
+import json
+import copy
+
+from openpype.client import get_project, create_project
+from openpype.settings import ProjectSettings, SaveWarningExc
+
+from openpype_modules.ftrack.lib import (
+ ServerAction,
+ get_openpype_attr,
+ CUST_ATTR_AUTO_SYNC
+)
+
+
+class PrepareProjectServer(ServerAction):
+ """Prepare project attributes in Anatomy."""
+
+ identifier = "prepare.project.server"
+ label = "OpenPype Admin"
+ variant = "- Prepare Project (Server)"
+ description = "Set basic attributes on the project"
+
+ settings_key = "prepare_project"
+
+ role_list = ["Pypeclub", "Administrator", "Project Manager"]
+
+ settings_key = "prepare_project"
+
+ item_splitter = {"type": "label", "value": "---"}
+ _keys_order = (
+ "fps",
+ "frameStart",
+ "frameEnd",
+ "handleStart",
+ "handleEnd",
+ "clipIn",
+ "clipOut",
+ "resolutionHeight",
+ "resolutionWidth",
+ "pixelAspect",
+ "applications",
+ "tools_env",
+ "library_project",
+ )
+
+ def discover(self, session, entities, event):
+ """Show only on project."""
+ if (
+ len(entities) != 1
+ or entities[0].entity_type.lower() != "project"
+ ):
+ return False
+
+ return self.valid_roles(session, entities, event)
+
+ def interface(self, session, entities, event):
+ if event['data'].get('values', {}):
+ return
+
+ # Inform user that this may take a while
+ self.show_message(event, "Preparing data... Please wait", True)
+ self.log.debug("Preparing data which will be shown")
+
+ self.log.debug("Loading custom attributes")
+
+ project_entity = entities[0]
+ project_name = project_entity["full_name"]
+
+ project_settings = ProjectSettings(project_name)
+
+ project_anatom_settings = project_settings["project_anatomy"]
+ root_items = self.prepare_root_items(project_anatom_settings)
+
+ ca_items, multiselect_enumerators = (
+ self.prepare_custom_attribute_items(project_anatom_settings)
+ )
+
+ self.log.debug("Heavy items are ready. Preparing last items group.")
+
+ title = "Prepare Project"
+ items = []
+
+ # Add root items
+ items.extend(root_items)
+
+ items.append(self.item_splitter)
+ items.append({
+ "type": "label",
+ "value": "Set basic Attributes: "
+ })
+
+ items.extend(ca_items)
+
+ # This item will be last before enumerators
+ # Set value of auto synchronization
+ auto_sync_value = project_entity["custom_attributes"].get(
+ CUST_ATTR_AUTO_SYNC, False
+ )
+ auto_sync_item = {
+ "name": CUST_ATTR_AUTO_SYNC,
+ "type": "boolean",
+ "value": auto_sync_value,
+ "label": "AutoSync to Avalon"
+ }
+ # Add autosync attribute
+ items.append(auto_sync_item)
+
+ # Add enumerator items at the end
+ for item in multiselect_enumerators:
+ items.append(item)
+
+ return {
+ "items": items,
+ "title": title
+ }
+
+ def prepare_root_items(self, project_anatom_settings):
+ self.log.debug("Root items preparation begins.")
+
+ root_items = []
+ root_items.append({
+ "type": "label",
+ "value": "Check your Project root settings "
+ })
+ root_items.append({
+ "type": "label",
+ "value": (
+ "NOTE: Roots are crutial for path filling"
+ " (and creating folder structure).
"
+ )
+ })
+ root_items.append({
+ "type": "label",
+ "value": (
+ "WARNING: Do not change roots on running project,"
+ " that will cause workflow issues .
"
+ )
+ })
+
+ empty_text = "Enter root path here..."
+
+ roots_entity = project_anatom_settings["roots"]
+ for root_name, root_entity in roots_entity.items():
+ root_items.append(self.item_splitter)
+ root_items.append({
+ "type": "label",
+ "value": "Root: \"{}\"".format(root_name)
+ })
+ for platform_name, value_entity in root_entity.items():
+ root_items.append({
+ "label": platform_name,
+ "name": "__root__{}__{}".format(root_name, platform_name),
+ "type": "text",
+ "value": value_entity.value,
+ "empty_text": empty_text
+ })
+
+ root_items.append({
+ "type": "hidden",
+ "name": "__rootnames__",
+ "value": json.dumps(list(roots_entity.keys()))
+ })
+
+ self.log.debug("Root items preparation ended.")
+ return root_items
+
+ def _attributes_to_set(self, project_anatom_settings):
+ attributes_to_set = {}
+
+ attribute_values_by_key = {}
+ for key, entity in project_anatom_settings["attributes"].items():
+ attribute_values_by_key[key] = entity.value
+
+ cust_attrs, hier_cust_attrs = get_openpype_attr(self.session, True)
+
+ for attr in hier_cust_attrs:
+ key = attr["key"]
+ if key.startswith("avalon_"):
+ continue
+ attributes_to_set[key] = {
+ "label": attr["label"],
+ "object": attr,
+ "default": attribute_values_by_key.get(key)
+ }
+
+ for attr in cust_attrs:
+ if attr["entity_type"].lower() != "show":
+ continue
+ key = attr["key"]
+ if key.startswith("avalon_"):
+ continue
+ attributes_to_set[key] = {
+ "label": attr["label"],
+ "object": attr,
+ "default": attribute_values_by_key.get(key)
+ }
+
+ # Sort by label
+ attributes_to_set = dict(sorted(
+ attributes_to_set.items(),
+ key=lambda x: x[1]["label"]
+ ))
+ return attributes_to_set
+
+ def prepare_custom_attribute_items(self, project_anatom_settings):
+ items = []
+ multiselect_enumerators = []
+ attributes_to_set = self._attributes_to_set(project_anatom_settings)
+
+ self.log.debug("Preparing interface for keys: \"{}\"".format(
+ str([key for key in attributes_to_set])
+ ))
+
+ attribute_keys = set(attributes_to_set.keys())
+ keys_order = []
+ for key in self._keys_order:
+ if key in attribute_keys:
+ keys_order.append(key)
+
+ attribute_keys = attribute_keys - set(keys_order)
+ for key in sorted(attribute_keys):
+ keys_order.append(key)
+
+ for key in keys_order:
+ in_data = attributes_to_set[key]
+ attr = in_data["object"]
+
+ # initial item definition
+ item = {
+ "name": key,
+ "label": in_data["label"]
+ }
+
+ # cust attr type - may have different visualization
+ type_name = attr["type"]["name"].lower()
+ easy_types = ["text", "boolean", "date", "number"]
+
+ easy_type = False
+ if type_name in easy_types:
+ easy_type = True
+
+ elif type_name == "enumerator":
+
+ attr_config = json.loads(attr["config"])
+ attr_config_data = json.loads(attr_config["data"])
+
+ if attr_config["multiSelect"] is True:
+ multiselect_enumerators.append(self.item_splitter)
+ multiselect_enumerators.append({
+ "type": "label",
+ "value": "{} ".format(in_data["label"])
+ })
+
+ default = in_data["default"]
+ names = []
+ for option in sorted(
+ attr_config_data, key=lambda x: x["menu"]
+ ):
+ name = option["value"]
+ new_name = "__{}__{}".format(key, name)
+ names.append(new_name)
+ item = {
+ "name": new_name,
+ "type": "boolean",
+ "label": "- {}".format(option["menu"])
+ }
+ if default:
+ if isinstance(default, (list, tuple)):
+ if name in default:
+ item["value"] = True
+ else:
+ if name == default:
+ item["value"] = True
+
+ multiselect_enumerators.append(item)
+
+ multiselect_enumerators.append({
+ "type": "hidden",
+ "name": "__hidden__{}".format(key),
+ "value": json.dumps(names)
+ })
+ else:
+ easy_type = True
+ item["data"] = attr_config_data
+
+ else:
+ self.log.warning((
+ "Custom attribute \"{}\" has type \"{}\"."
+ " I don't know how to handle"
+ ).format(key, type_name))
+ items.append({
+ "type": "label",
+ "value": (
+ "!!! Can't handle Custom attritubte type \"{}\""
+ " (key: \"{}\")"
+ ).format(type_name, key)
+ })
+
+ if easy_type:
+ item["type"] = type_name
+
+ # default value in interface
+ default = in_data["default"]
+ if default is not None:
+ item["value"] = default
+
+ items.append(item)
+
+ return items, multiselect_enumerators
+
+ def launch(self, session, entities, event):
+ in_data = event["data"].get("values")
+ if not in_data:
+ return
+
+ root_values = {}
+ root_key = "__root__"
+ for key in tuple(in_data.keys()):
+ if key.startswith(root_key):
+ _key = key[len(root_key):]
+ root_values[_key] = in_data.pop(key)
+
+ root_names = in_data.pop("__rootnames__", None)
+ root_data = {}
+ for root_name in json.loads(root_names):
+ root_data[root_name] = {}
+ for key, value in tuple(root_values.items()):
+ prefix = "{}__".format(root_name)
+ if not key.startswith(prefix):
+ continue
+
+ _key = key[len(prefix):]
+ root_data[root_name][_key] = value
+
+ # Find hidden items for multiselect enumerators
+ keys_to_process = []
+ for key in in_data:
+ if key.startswith("__hidden__"):
+ keys_to_process.append(key)
+
+ self.log.debug("Preparing data for Multiselect Enumerators")
+ enumerators = {}
+ for key in keys_to_process:
+ new_key = key.replace("__hidden__", "")
+ enumerator_items = in_data.pop(key)
+ enumerators[new_key] = json.loads(enumerator_items)
+
+ # find values set for multiselect enumerator
+ for key, enumerator_items in enumerators.items():
+ in_data[key] = []
+
+ name = "__{}__".format(key)
+
+ for item in enumerator_items:
+ value = in_data.pop(item)
+ if value is True:
+ new_key = item.replace(name, "")
+ in_data[key].append(new_key)
+
+ self.log.debug("Setting Custom Attribute values")
+
+ project_entity = entities[0]
+ project_name = project_entity["full_name"]
+
+ # Try to find project document
+ project_doc = get_project(project_name)
+
+ # Create project if is not available
+ # - creation is required to be able set project anatomy and attributes
+ if not project_doc:
+ project_code = project_entity["name"]
+ self.log.info("Creating project \"{} [{}]\"".format(
+ project_name, project_code
+ ))
+ create_project(project_name, project_code)
+ self.trigger_event(
+ "openpype.project.created",
+ {"project_name": project_name}
+ )
+
+ project_settings = ProjectSettings(project_name)
+ project_anatomy_settings = project_settings["project_anatomy"]
+ project_anatomy_settings["roots"] = root_data
+
+ custom_attribute_values = {}
+ attributes_entity = project_anatomy_settings["attributes"]
+ for key, value in in_data.items():
+ if key not in attributes_entity:
+ custom_attribute_values[key] = value
+ else:
+ attributes_entity[key] = value
+
+ try:
+ project_settings.save()
+ except SaveWarningExc as exc:
+ self.log.info("Few warnings happened during settings save:")
+ for warning in exc.warnings:
+ self.log.info(str(warning))
+
+ # Change custom attributes on project
+ if custom_attribute_values:
+ for key, value in custom_attribute_values.items():
+ project_entity["custom_attributes"][key] = value
+ self.log.debug("- Key \"{}\" set to \"{}\"".format(key, value))
+ session.commit()
+
+ event_data = copy.deepcopy(in_data)
+ event_data["project_name"] = project_name
+ self.trigger_event("openpype.project.prepared", event_data)
+
+ return True
+
+
+def register(session):
+ '''Register plugin. Called when used as an plugin.'''
+ PrepareProjectServer(session).register()
diff --git a/openpype/modules/default_modules/ftrack/event_handlers_server/action_private_project_detection.py b/openpype/modules/ftrack/event_handlers_server/action_private_project_detection.py
similarity index 100%
rename from openpype/modules/default_modules/ftrack/event_handlers_server/action_private_project_detection.py
rename to openpype/modules/ftrack/event_handlers_server/action_private_project_detection.py
diff --git a/openpype/modules/default_modules/ftrack/event_handlers_server/action_push_frame_values_to_task.py b/openpype/modules/ftrack/event_handlers_server/action_push_frame_values_to_task.py
similarity index 95%
rename from openpype/modules/default_modules/ftrack/event_handlers_server/action_push_frame_values_to_task.py
rename to openpype/modules/ftrack/event_handlers_server/action_push_frame_values_to_task.py
index 3f63ce6face..1209375f821 100644
--- a/openpype/modules/default_modules/ftrack/event_handlers_server/action_push_frame_values_to_task.py
+++ b/openpype/modules/ftrack/event_handlers_server/action_push_frame_values_to_task.py
@@ -2,7 +2,10 @@
import json
import collections
import ftrack_api
-from openpype_modules.ftrack.lib import ServerAction
+from openpype_modules.ftrack.lib import (
+ ServerAction,
+ query_custom_attributes
+)
class PushHierValuesToNonHier(ServerAction):
@@ -51,10 +54,6 @@ class PushHierValuesToNonHier(ServerAction):
" from CustomAttributeConfiguration"
" where key in ({})"
)
- cust_attr_value_query = (
- "select value, entity_id from CustomAttributeValue"
- " where entity_id in ({}) and configuration_id in ({})"
- )
# configurable
settings_key = "sync_hier_entity_attributes"
@@ -344,25 +343,11 @@ def get_hier_values(
all_ids_with_parents.add(parent_id)
_entity_id = parent_id
- joined_entity_ids = self.join_query_keys(all_ids_with_parents)
-
- hier_attr_ids = self.join_query_keys(
- tuple(hier_attr["id"] for hier_attr in hier_attrs)
- )
+ hier_attr_ids = tuple(hier_attr["id"] for hier_attr in hier_attrs)
hier_attrs_key_by_id = {
hier_attr["id"]: hier_attr["key"]
for hier_attr in hier_attrs
}
- call_expr = [{
- "action": "query",
- "expression": self.cust_attr_value_query.format(
- joined_entity_ids, hier_attr_ids
- )
- }]
- if hasattr(session, "call"):
- [values] = session.call(call_expr)
- else:
- [values] = session._call(call_expr)
values_per_entity_id = {}
for entity_id in all_ids_with_parents:
@@ -370,7 +355,10 @@ def get_hier_values(
for key in hier_attrs_key_by_id.values():
values_per_entity_id[entity_id][key] = None
- for item in values["data"]:
+ values = query_custom_attributes(
+ session, hier_attr_ids, all_ids_with_parents, True
+ )
+ for item in values:
entity_id = item["entity_id"]
key = hier_attrs_key_by_id[item["configuration_id"]]
diff --git a/openpype/modules/ftrack/event_handlers_server/action_sync_to_avalon.py b/openpype/modules/ftrack/event_handlers_server/action_sync_to_avalon.py
new file mode 100644
index 00000000000..df9147bdf77
--- /dev/null
+++ b/openpype/modules/ftrack/event_handlers_server/action_sync_to_avalon.py
@@ -0,0 +1,220 @@
+import time
+import sys
+import json
+
+import ftrack_api
+
+from openpype_modules.ftrack.lib import ServerAction
+from openpype_modules.ftrack.lib.avalon_sync import SyncEntitiesFactory
+
+
+class SyncToAvalonServer(ServerAction):
+ """
+ Synchronizing data action - from Ftrack to Avalon DB
+
+ Stores all information about entity.
+ - Name(string) - Most important information = identifier of entity
+ - Parent(ObjectId) - Avalon Project Id, if entity is not project itself
+ - Data(dictionary):
+ - VisualParent(ObjectId) - Avalon Id of parent asset
+ - Parents(array of string) - All parent names except project
+ - Tasks(dictionary of dictionaries) - Tasks on asset
+ - FtrackId(string)
+ - entityType(string) - entity's type on Ftrack
+ * All Custom attributes in group 'Avalon'
+ - custom attributes that start with 'avalon_' are skipped
+
+ * These information are stored for entities in whole project.
+
+ Avalon ID of asset is stored to Ftrack
+ - Custom attribute 'avalon_mongo_id'.
+ - action IS NOT creating this Custom attribute if doesn't exist
+ - run 'Create Custom Attributes' action
+ - or do it manually (Not recommended)
+ """
+ #: Action identifier.
+ identifier = "sync.to.avalon.server"
+ #: Action label.
+ label = "OpenPype Admin"
+ variant = "- Sync To Avalon (Server)"
+ #: Action description.
+ description = "Send data from Ftrack to Avalon"
+ role_list = {"Pypeclub", "Administrator", "Project Manager"}
+
+ def __init__(self, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ self.entities_factory = SyncEntitiesFactory(self.log, self.session)
+
+ def discover(self, session, entities, event):
+ """ Validation """
+ # Check if selection is valid
+ for ent in event["data"]["selection"]:
+ # Ignore entities that are not tasks or projects
+ if ent["entityType"].lower() in ["show", "task"]:
+ return True
+ return False
+
+ def launch(self, session, in_entities, event):
+ self.log.debug("{}: Creating job".format(self.label))
+
+ user_entity = session.query(
+ "User where id is {}".format(event["source"]["user"]["id"])
+ ).one()
+ job_entity = session.create("Job", {
+ "user": user_entity,
+ "status": "running",
+ "data": json.dumps({
+ "description": "Sync to avalon is running..."
+ })
+ })
+ session.commit()
+
+ project_entity = self.get_project_from_entity(in_entities[0])
+ project_name = project_entity["full_name"]
+
+ try:
+ result = self.synchronization(event, project_name)
+
+ except Exception:
+ self.log.error(
+ "Synchronization failed due to code error", exc_info=True
+ )
+
+ description = "Sync to avalon Crashed (Download traceback)"
+ self.add_traceback_to_job(
+ job_entity, session, sys.exc_info(), description
+ )
+
+ msg = "An error has happened during synchronization"
+ title = "Synchronization report ({}):".format(project_name)
+ items = []
+ items.append({
+ "type": "label",
+ "value": "# {}".format(msg)
+ })
+ items.append({
+ "type": "label",
+ "value": (
+ "Download report from job for more information.
"
+ )
+ })
+
+ report = {}
+ try:
+ report = self.entities_factory.report()
+ except Exception:
+ pass
+
+ _items = report.get("items") or []
+ if _items:
+ items.append(self.entities_factory.report_splitter)
+ items.extend(_items)
+
+ self.show_interface(items, title, event, submit_btn_label="Ok")
+
+ return {"success": True, "message": msg}
+
+ job_entity["status"] = "done"
+ job_entity["data"] = json.dumps({
+ "description": "Sync to avalon finished."
+ })
+ session.commit()
+
+ return result
+
+ def synchronization(self, event, project_name):
+ time_start = time.time()
+
+ self.show_message(event, "Synchronization - Preparing data", True)
+
+ try:
+ output = self.entities_factory.launch_setup(project_name)
+ if output is not None:
+ return output
+
+ time_1 = time.time()
+
+ self.entities_factory.set_cutom_attributes()
+ time_2 = time.time()
+
+ # This must happen before all filtering!!!
+ self.entities_factory.prepare_avalon_entities(project_name)
+ time_3 = time.time()
+
+ self.entities_factory.filter_by_ignore_sync()
+ time_4 = time.time()
+
+ self.entities_factory.duplicity_regex_check()
+ time_5 = time.time()
+
+ self.entities_factory.prepare_ftrack_ent_data()
+ time_6 = time.time()
+
+ self.entities_factory.synchronize()
+ time_7 = time.time()
+
+ self.log.debug(
+ "*** Synchronization finished ***"
+ )
+ self.log.debug(
+ "preparation <{}>".format(time_1 - time_start)
+ )
+ self.log.debug(
+ "set_cutom_attributes <{}>".format(time_2 - time_1)
+ )
+ self.log.debug(
+ "prepare_avalon_entities <{}>".format(time_3 - time_2)
+ )
+ self.log.debug(
+ "filter_by_ignore_sync <{}>".format(time_4 - time_3)
+ )
+ self.log.debug(
+ "duplicity_regex_check <{}>".format(time_5 - time_4)
+ )
+ self.log.debug(
+ "prepare_ftrack_ent_data <{}>".format(time_6 - time_5)
+ )
+ self.log.debug(
+ "synchronize <{}>".format(time_7 - time_6)
+ )
+ self.log.debug(
+ "* Total time: {}".format(time_7 - time_start)
+ )
+
+ if self.entities_factory.project_created:
+ event = ftrack_api.event.base.Event(
+ topic="openpype.project.created",
+ data={"project_name": project_name}
+ )
+ self.session.event_hub.publish(event)
+
+ report = self.entities_factory.report()
+ if report and report.get("items"):
+ default_title = "Synchronization report ({}):".format(
+ project_name
+ )
+ self.show_interface(
+ items=report["items"],
+ title=report.get("title", default_title),
+ event=event
+ )
+ return {
+ "success": True,
+ "message": "Synchronization Finished"
+ }
+
+ finally:
+ try:
+ self.entities_factory.dbcon.uninstall()
+ except Exception:
+ pass
+
+ try:
+ self.entities_factory.session.close()
+ except Exception:
+ pass
+
+
+def register(session):
+ '''Register plugin. Called when used as an plugin.'''
+ SyncToAvalonServer(session).register()
diff --git a/openpype/modules/ftrack/event_handlers_server/action_tranfer_hierarchical_values.py b/openpype/modules/ftrack/event_handlers_server/action_tranfer_hierarchical_values.py
new file mode 100644
index 00000000000..d160b7200db
--- /dev/null
+++ b/openpype/modules/ftrack/event_handlers_server/action_tranfer_hierarchical_values.py
@@ -0,0 +1,346 @@
+import copy
+import json
+import collections
+
+import ftrack_api
+
+from openpype_modules.ftrack.lib import (
+ ServerAction,
+ statics_icon,
+)
+from openpype_modules.ftrack.lib.avalon_sync import create_chunks
+
+
+class TransferHierarchicalValues(ServerAction):
+ """Transfer values across hierarhcical attributes.
+
+ Aalso gives ability to convert types meanwhile. That is limited to
+ conversions between numbers and strings
+ - int <-> float
+ - in, float -> string
+ """
+
+ identifier = "transfer.hierarchical.values"
+ label = "OpenPype Admin"
+ variant = "- Transfer values between 2 custom attributes"
+ description = (
+ "Move values from a hierarchical attribute to"
+ " second hierarchical attribute."
+ )
+ icon = statics_icon("ftrack", "action_icons", "OpenPypeAdmin.svg")
+
+ all_project_entities_query = (
+ "select id, name, parent_id, link"
+ " from TypedContext where project_id is \"{}\""
+ )
+ cust_attr_query = (
+ "select value, entity_id from CustomAttributeValue"
+ " where entity_id in ({}) and configuration_id is \"{}\""
+ )
+ settings_key = "transfer_values_of_hierarchical_attributes"
+
+ def discover(self, session, entities, event):
+ """Show anywhere."""
+
+ return self.valid_roles(session, entities, event)
+
+ def _selection_interface(self, session, event_values=None):
+ title = "Transfer hierarchical values"
+
+ attr_confs = session.query(
+ (
+ "select id, key from CustomAttributeConfiguration"
+ " where is_hierarchical is true"
+ )
+ ).all()
+ attr_items = []
+ for attr_conf in attr_confs:
+ attr_items.append({
+ "value": attr_conf["id"],
+ "label": attr_conf["key"]
+ })
+
+ if len(attr_items) < 2:
+ return {
+ "title": title,
+ "items": [{
+ "type": "label",
+ "value": (
+ "Didn't found custom attributes"
+ " that can be transfered."
+ )
+ }]
+ }
+
+ attr_items = sorted(attr_items, key=lambda item: item["label"])
+ items = []
+ item_splitter = {"type": "label", "value": "---"}
+ items.append({
+ "type": "label",
+ "value": (
+ "Please select source and destination"
+ " Custom attribute "
+ )
+ })
+ items.append({
+ "type": "label",
+ "value": (
+ "WARNING: This will take affect for all projects!"
+ )
+ })
+ if event_values:
+ items.append({
+ "type": "label",
+ "value": (
+ "Note: Please select 2 different custom attributes."
+ )
+ })
+
+ items.append(item_splitter)
+
+ src_item = {
+ "type": "enumerator",
+ "label": "Source",
+ "name": "src_attr_id",
+ "data": copy.deepcopy(attr_items)
+ }
+ dst_item = {
+ "type": "enumerator",
+ "label": "Destination",
+ "name": "dst_attr_id",
+ "data": copy.deepcopy(attr_items)
+ }
+ delete_item = {
+ "type": "boolean",
+ "name": "delete_dst_attr_first",
+ "label": "Delete first",
+ "value": False
+ }
+ if event_values:
+ src_item["value"] = event_values["src_attr_id"]
+ dst_item["value"] = event_values["dst_attr_id"]
+ delete_item["value"] = event_values["delete_dst_attr_first"]
+
+ items.append(src_item)
+ items.append(dst_item)
+ items.append(item_splitter)
+ items.append({
+ "type": "label",
+ "value": (
+ "WARNING: All values from destination"
+ " Custom Attribute will be removed if this is enabled."
+ )
+ })
+ items.append(delete_item)
+
+ return {
+ "title": title,
+ "items": items
+ }
+
+ def interface(self, session, entities, event):
+ if event["data"].get("values", {}):
+ return None
+
+ return self._selection_interface(session)
+
+ def launch(self, session, entities, event):
+ values = event["data"].get("values", {})
+ if not values:
+ return None
+ src_attr_id = values["src_attr_id"]
+ dst_attr_id = values["dst_attr_id"]
+ delete_dst_values = values["delete_dst_attr_first"]
+
+ if not src_attr_id or not dst_attr_id:
+ self.log.info("Attributes were not filled. Nothing to do.")
+ return {
+ "success": True,
+ "message": "Nothing to do"
+ }
+
+ if src_attr_id == dst_attr_id:
+ self.log.info((
+ "Same attributes were selected {}, {}."
+ " Showing interface again."
+ ).format(src_attr_id, dst_attr_id))
+ return self._selection_interface(session, values)
+
+ # Query custom attrbutes
+ src_conf = session.query((
+ "select id from CustomAttributeConfiguration where id is {}"
+ ).format(src_attr_id)).one()
+ dst_conf = session.query((
+ "select id from CustomAttributeConfiguration where id is {}"
+ ).format(dst_attr_id)).one()
+ src_type_name = src_conf["type"]["name"]
+ dst_type_name = dst_conf["type"]["name"]
+ # Limit conversion to
+ # - same type -> same type (there is no need to do conversion)
+ # - number -> number (int to float and back)
+ # - number -> str (any number can be converted to str)
+ src_type = None
+ dst_type = None
+ if src_type_name == "number" or src_type_name != dst_type_name:
+ src_type = self._get_attr_type(dst_conf)
+ dst_type = self._get_attr_type(dst_conf)
+ valid = False
+ # Can convert numbers
+ if src_type in (int, float) and dst_type in (int, float):
+ valid = True
+ # Can convert numbers to string
+ elif dst_type is str:
+ valid = True
+
+ if not valid:
+ self.log.info((
+ "Don't know how to properly convert"
+ " custom attribute types {} > {}"
+ ).format(src_type_name, dst_type_name))
+ return {
+ "message": (
+ "Don't know how to properly convert"
+ " custom attribute types {} > {}"
+ ).format(src_type_name, dst_type_name),
+ "success": False
+ }
+
+ # Query source values
+ src_attr_values = session.query(
+ (
+ "select value, entity_id"
+ " from CustomAttributeValue"
+ " where configuration_id is {}"
+ ).format(src_attr_id)
+ ).all()
+
+ self.log.debug("Queried source values.")
+ failed_entity_ids = []
+ if dst_type is not None:
+ self.log.debug("Converting source values to desctination type")
+ value_by_id = {}
+ for attr_value in src_attr_values:
+ entity_id = attr_value["entity_id"]
+ value = attr_value["value"]
+ if value is not None:
+ try:
+ if dst_type is not None:
+ value = dst_type(value)
+ value_by_id[entity_id] = value
+ except Exception:
+ failed_entity_ids.append(entity_id)
+
+ if failed_entity_ids:
+ self.log.info(
+ "Couldn't convert some values to destination attribute"
+ )
+ return {
+ "success": False,
+ "message": (
+ "Couldn't convert some values to destination attribute"
+ )
+ }
+
+ # Delete destination custom attributes first
+ if delete_dst_values:
+ self.log.info("Deleting destination custom attribute values first")
+ self._delete_custom_attribute_values(session, dst_attr_id)
+
+ self.log.info("Applying source values on destination custom attribute")
+ self._apply_values(session, value_by_id, dst_attr_id)
+ return True
+
+ def _delete_custom_attribute_values(self, session, dst_attr_id):
+ dst_attr_values = session.query(
+ (
+ "select configuration_id, entity_id"
+ " from CustomAttributeValue"
+ " where configuration_id is {}"
+ ).format(dst_attr_id)
+ ).all()
+ delete_operations = []
+ for attr_value in dst_attr_values:
+ entity_id = attr_value["entity_id"]
+ configuration_id = attr_value["configuration_id"]
+ entity_key = collections.OrderedDict((
+ ("configuration_id", configuration_id),
+ ("entity_id", entity_id)
+ ))
+ delete_operations.append(
+ ftrack_api.operation.DeleteEntityOperation(
+ "CustomAttributeValue",
+ entity_key
+ )
+ )
+
+ if not delete_operations:
+ return
+
+ for chunk in create_chunks(delete_operations, 500):
+ for operation in chunk:
+ session.recorded_operations.push(operation)
+ session.commit()
+
+ def _apply_values(self, session, value_by_id, dst_attr_id):
+ dst_attr_values = session.query(
+ (
+ "select configuration_id, entity_id"
+ " from CustomAttributeValue"
+ " where configuration_id is {}"
+ ).format(dst_attr_id)
+ ).all()
+
+ dst_entity_ids_with_value = {
+ item["entity_id"]
+ for item in dst_attr_values
+ }
+ operations = []
+ for entity_id, value in value_by_id.items():
+ entity_key = collections.OrderedDict((
+ ("configuration_id", dst_attr_id),
+ ("entity_id", entity_id)
+ ))
+ if entity_id in dst_entity_ids_with_value:
+ operations.append(
+ ftrack_api.operation.UpdateEntityOperation(
+ "CustomAttributeValue",
+ entity_key,
+ "value",
+ ftrack_api.symbol.NOT_SET,
+ value
+ )
+ )
+ else:
+ operations.append(
+ ftrack_api.operation.CreateEntityOperation(
+ "CustomAttributeValue",
+ entity_key,
+ {"value": value}
+ )
+ )
+
+ if not operations:
+ return
+
+ for chunk in create_chunks(operations, 500):
+ for operation in chunk:
+ session.recorded_operations.push(operation)
+ session.commit()
+
+ def _get_attr_type(self, conf_def):
+ type_name = conf_def["type"]["name"]
+ if type_name == "text":
+ return str
+
+ if type_name == "number":
+ config = json.loads(conf_def["config"])
+ if config["isdecimal"]:
+ return float
+ return int
+ return None
+
+
+def register(session):
+ '''Register plugin. Called when used as an plugin.'''
+
+ TransferHierarchicalValues(session).register()
diff --git a/openpype/modules/default_modules/ftrack/event_handlers_server/event_del_avalon_id_from_new.py b/openpype/modules/ftrack/event_handlers_server/event_del_avalon_id_from_new.py
similarity index 100%
rename from openpype/modules/default_modules/ftrack/event_handlers_server/event_del_avalon_id_from_new.py
rename to openpype/modules/ftrack/event_handlers_server/event_del_avalon_id_from_new.py
diff --git a/openpype/modules/default_modules/ftrack/event_handlers_server/event_first_version_status.py b/openpype/modules/ftrack/event_handlers_server/event_first_version_status.py
similarity index 100%
rename from openpype/modules/default_modules/ftrack/event_handlers_server/event_first_version_status.py
rename to openpype/modules/ftrack/event_handlers_server/event_first_version_status.py
diff --git a/openpype/modules/default_modules/ftrack/event_handlers_server/event_next_task_update.py b/openpype/modules/ftrack/event_handlers_server/event_next_task_update.py
similarity index 100%
rename from openpype/modules/default_modules/ftrack/event_handlers_server/event_next_task_update.py
rename to openpype/modules/ftrack/event_handlers_server/event_next_task_update.py
diff --git a/openpype/modules/default_modules/ftrack/event_handlers_server/event_push_frame_values_to_task.py b/openpype/modules/ftrack/event_handlers_server/event_push_frame_values_to_task.py
similarity index 76%
rename from openpype/modules/default_modules/ftrack/event_handlers_server/event_push_frame_values_to_task.py
rename to openpype/modules/ftrack/event_handlers_server/event_push_frame_values_to_task.py
index 10b165e7f6c..dc76920a57c 100644
--- a/openpype/modules/default_modules/ftrack/event_handlers_server/event_push_frame_values_to_task.py
+++ b/openpype/modules/ftrack/event_handlers_server/event_push_frame_values_to_task.py
@@ -1,10 +1,11 @@
import collections
import datetime
+import copy
import ftrack_api
from openpype_modules.ftrack.lib import (
BaseEvent,
- query_custom_attributes
+ query_custom_attributes,
)
@@ -17,11 +18,6 @@ class PushFrameValuesToTaskEvent(BaseEvent):
" (object_type_id in ({}) or is_hierarchical is true)"
)
- cust_attr_query = (
- "select value, entity_id from ContextCustomAttributeValue "
- "where entity_id in ({}) and configuration_id in ({})"
- )
-
_cached_task_object_id = None
_cached_interest_object_ids = None
_cached_user_id = None
@@ -129,10 +125,15 @@ def process_by_project(self, session, event, project_id, entities_info):
# Separate value changes and task parent changes
_entities_info = []
+ added_entities = []
+ added_entity_ids = set()
task_parent_changes = []
for entity_info in entities_info:
if entity_info["entity_type"].lower() == "task":
task_parent_changes.append(entity_info)
+ elif entity_info.get("action") == "add":
+ added_entities.append(entity_info)
+ added_entity_ids.add(entity_info["entityId"])
else:
_entities_info.append(entity_info)
entities_info = _entities_info
@@ -141,6 +142,13 @@ def process_by_project(self, session, event, project_id, entities_info):
interesting_data, changed_keys_by_object_id = self.filter_changes(
session, event, entities_info, interest_attributes
)
+ self.interesting_data_for_added(
+ session,
+ added_entities,
+ interest_attributes,
+ interesting_data,
+ changed_keys_by_object_id
+ )
if not interesting_data and not task_parent_changes:
return
@@ -156,9 +164,13 @@ def process_by_project(self, session, event, project_id, entities_info):
# - it is a complex way how to find out
if interesting_data:
self.process_attribute_changes(
- session, object_types_by_name,
- interesting_data, changed_keys_by_object_id,
- interest_entity_types, interest_attributes
+ session,
+ object_types_by_name,
+ interesting_data,
+ changed_keys_by_object_id,
+ interest_entity_types,
+ interest_attributes,
+ added_entity_ids
)
if task_parent_changes:
@@ -168,8 +180,12 @@ def process_by_project(self, session, event, project_id, entities_info):
)
def process_task_parent_change(
- self, session, object_types_by_name, task_parent_changes,
- interest_entity_types, interest_attributes
+ self,
+ session,
+ object_types_by_name,
+ task_parent_changes,
+ interest_entity_types,
+ interest_attributes
):
"""Push custom attribute values if task parent has changed.
@@ -181,6 +197,7 @@ def process_task_parent_change(
real hierarchical value and non hierarchical custom attribute value
should be set to hierarchical value.
"""
+
# Store task ids which were created or moved under parent with entity
# type defined in settings (interest_entity_types).
task_ids = set()
@@ -273,16 +290,23 @@ def process_task_parent_change(
hier_attr_ids.append(attr_id)
conf_ids = list(hier_attr_ids)
+ task_conf_ids = []
for key, attr_id in task_attrs.items():
attr_key_by_id[attr_id] = key
nonhier_id_by_key[key] = attr_id
conf_ids.append(attr_id)
+ task_conf_ids.append(attr_id)
# Query custom attribute values
# - result does not contain values for all entities only result of
# query callback to ftrack server
result = query_custom_attributes(
- session, conf_ids, whole_hierarchy_ids
+ session, list(hier_attr_ids), whole_hierarchy_ids, True
+ )
+ result.extend(
+ query_custom_attributes(
+ session, task_conf_ids, whole_hierarchy_ids, False
+ )
)
# Prepare variables where result will be stored
@@ -378,33 +402,49 @@ def _commit_changes(self, session, changes):
uncommited_changes = False
for idx, item in enumerate(changes):
new_value = item["new_value"]
+ old_value = item["old_value"]
attr_id = item["attr_id"]
entity_id = item["entity_id"]
attr_key = item["attr_key"]
- entity_key = collections.OrderedDict()
- entity_key["configuration_id"] = attr_id
- entity_key["entity_id"] = entity_id
+ entity_key = collections.OrderedDict((
+ ("configuration_id", attr_id),
+ ("entity_id", entity_id)
+ ))
self._cached_changes.append({
"attr_key": attr_key,
"entity_id": entity_id,
"value": new_value,
"time": datetime.datetime.now()
})
+ old_value_is_set = (
+ old_value is not ftrack_api.symbol.NOT_SET
+ and old_value is not None
+ )
if new_value is None:
+ if not old_value_is_set:
+ continue
op = ftrack_api.operation.DeleteEntityOperation(
"CustomAttributeValue",
entity_key
)
- else:
+
+ elif old_value_is_set:
op = ftrack_api.operation.UpdateEntityOperation(
- "ContextCustomAttributeValue",
+ "CustomAttributeValue",
entity_key,
"value",
- ftrack_api.symbol.NOT_SET,
+ old_value,
new_value
)
+ else:
+ op = ftrack_api.operation.CreateEntityOperation(
+ "CustomAttributeValue",
+ entity_key,
+ {"value": new_value}
+ )
+
session.recorded_operations.push(op)
self.log.info((
"Changing Custom Attribute \"{}\" to value"
@@ -430,9 +470,14 @@ def _commit_changes(self, session, changes):
self.log.warning("Changing of values failed.", exc_info=True)
def process_attribute_changes(
- self, session, object_types_by_name,
- interesting_data, changed_keys_by_object_id,
- interest_entity_types, interest_attributes
+ self,
+ session,
+ object_types_by_name,
+ interesting_data,
+ changed_keys_by_object_id,
+ interest_entity_types,
+ interest_attributes,
+ added_entity_ids
):
# Prepare task object id
task_object_id = object_types_by_name["task"]["id"]
@@ -520,15 +565,26 @@ def process_attribute_changes(
parent_id_by_task_id[task_id] = task_entity["parent_id"]
self.finalize_attribute_changes(
- session, interesting_data,
- changed_keys, attrs_by_obj_id, hier_attrs,
- task_entity_ids, parent_id_by_task_id
+ session,
+ interesting_data,
+ changed_keys,
+ attrs_by_obj_id,
+ hier_attrs,
+ task_entity_ids,
+ parent_id_by_task_id,
+ added_entity_ids
)
def finalize_attribute_changes(
- self, session, interesting_data,
- changed_keys, attrs_by_obj_id, hier_attrs,
- task_entity_ids, parent_id_by_task_id
+ self,
+ session,
+ interesting_data,
+ changed_keys,
+ attrs_by_obj_id,
+ hier_attrs,
+ task_entity_ids,
+ parent_id_by_task_id,
+ added_entity_ids
):
attr_id_to_key = {}
for attr_confs in attrs_by_obj_id.values():
@@ -547,8 +603,12 @@ def finalize_attribute_changes(
)
attr_ids = set(attr_id_to_key.keys())
- current_values_by_id = self.current_values(
- session, attr_ids, entity_ids, task_entity_ids, hier_attrs
+ current_values_by_id = self.get_current_values(
+ session,
+ attr_ids,
+ entity_ids,
+ task_entity_ids,
+ hier_attrs
)
changes = []
@@ -558,14 +618,25 @@ def finalize_attribute_changes(
parent_id = entity_id
values = interesting_data[parent_id]
+ added_entity = entity_id in added_entity_ids
for attr_id, old_value in current_values.items():
+ if added_entity and attr_id in hier_attrs:
+ continue
+
attr_key = attr_id_to_key.get(attr_id)
if not attr_key:
continue
# Convert new value from string
new_value = values.get(attr_key)
- if new_value is not None and old_value is not None:
+ new_value_is_valid = (
+ old_value is not ftrack_api.symbol.NOT_SET
+ and new_value is not None
+ )
+ if added_entity and not new_value_is_valid:
+ continue
+
+ if new_value is not None and new_value_is_valid:
try:
new_value = type(old_value)(new_value)
except Exception:
@@ -579,6 +650,7 @@ def finalize_attribute_changes(
changes.append({
"new_value": new_value,
"attr_id": attr_id,
+ "old_value": old_value,
"entity_id": entity_id,
"attr_key": attr_key
})
@@ -597,6 +669,7 @@ def filter_changes(
interesting_data = {}
changed_keys_by_object_id = {}
+
for entity_info in entities_info:
# Care only about changes if specific keys
entity_changes = {}
@@ -642,27 +715,124 @@ def filter_changes(
return interesting_data, changed_keys_by_object_id
- def current_values(
- self, session, attr_ids, entity_ids, task_entity_ids, hier_attrs
+ def interesting_data_for_added(
+ self,
+ session,
+ added_entities,
+ interest_attributes,
+ interesting_data,
+ changed_keys_by_object_id
+ ):
+ if not added_entities or not interest_attributes:
+ return
+
+ object_type_ids = set()
+ entity_ids = set()
+ all_entity_ids = set()
+ object_id_by_entity_id = {}
+ project_id = None
+ entity_ids_by_parent_id = collections.defaultdict(set)
+ for entity_info in added_entities:
+ object_id = entity_info["objectTypeId"]
+ entity_id = entity_info["entityId"]
+ object_type_ids.add(object_id)
+ entity_ids.add(entity_id)
+ object_id_by_entity_id[entity_id] = object_id
+
+ for item in entity_info["parents"]:
+ entity_id = item["entityId"]
+ all_entity_ids.add(entity_id)
+ parent_id = item["parentId"]
+ if not parent_id:
+ project_id = entity_id
+ else:
+ entity_ids_by_parent_id[parent_id].add(entity_id)
+
+ hier_attrs = self.get_hierarchical_configurations(
+ session, interest_attributes
+ )
+ if not hier_attrs:
+ return
+
+ hier_attrs_key_by_id = {
+ attr_conf["id"]: attr_conf["key"]
+ for attr_conf in hier_attrs
+ }
+ default_values_by_key = {
+ attr_conf["key"]: attr_conf["default"]
+ for attr_conf in hier_attrs
+ }
+
+ values = query_custom_attributes(
+ session, list(hier_attrs_key_by_id.keys()), all_entity_ids, True
+ )
+ values_per_entity_id = {}
+ for entity_id in all_entity_ids:
+ values_per_entity_id[entity_id] = {}
+ for attr_name in interest_attributes:
+ values_per_entity_id[entity_id][attr_name] = None
+
+ for item in values:
+ entity_id = item["entity_id"]
+ key = hier_attrs_key_by_id[item["configuration_id"]]
+ values_per_entity_id[entity_id][key] = item["value"]
+
+ fill_queue = collections.deque()
+ fill_queue.append((project_id, default_values_by_key))
+ while fill_queue:
+ item = fill_queue.popleft()
+ entity_id, values_by_key = item
+ entity_values = values_per_entity_id[entity_id]
+ new_values_by_key = copy.deepcopy(values_by_key)
+ for key, value in values_by_key.items():
+ current_value = entity_values[key]
+ if current_value is None:
+ entity_values[key] = value
+ else:
+ new_values_by_key[key] = current_value
+
+ for child_id in entity_ids_by_parent_id[entity_id]:
+ fill_queue.append((child_id, new_values_by_key))
+
+ for entity_id in entity_ids:
+ entity_changes = {}
+ for key, value in values_per_entity_id[entity_id].items():
+ if value is not None:
+ entity_changes[key] = value
+
+ if not entity_changes:
+ continue
+
+ interesting_data[entity_id] = entity_changes
+ object_id = object_id_by_entity_id[entity_id]
+ if object_id not in changed_keys_by_object_id:
+ changed_keys_by_object_id[object_id] = set()
+ changed_keys_by_object_id[object_id] |= set(entity_changes.keys())
+
+ def get_current_values(
+ self,
+ session,
+ attr_ids,
+ entity_ids,
+ task_entity_ids,
+ hier_attrs
):
current_values_by_id = {}
if not attr_ids or not entity_ids:
return current_values_by_id
- joined_conf_ids = self.join_query_keys(attr_ids)
- joined_entity_ids = self.join_query_keys(entity_ids)
- call_expr = [{
- "action": "query",
- "expression": self.cust_attr_query.format(
- joined_entity_ids, joined_conf_ids
- )
- }]
- if hasattr(session, "call"):
- [values] = session.call(call_expr)
- else:
- [values] = session._call(call_expr)
+ for entity_id in entity_ids:
+ current_values_by_id[entity_id] = {}
+ for attr_id in attr_ids:
+ current_values_by_id[entity_id][attr_id] = (
+ ftrack_api.symbol.NOT_SET
+ )
+
+ values = query_custom_attributes(
+ session, attr_ids, entity_ids, True
+ )
- for item in values["data"]:
+ for item in values:
entity_id = item["entity_id"]
attr_id = item["configuration_id"]
if entity_id in task_entity_ids and attr_id in hier_attrs:
@@ -707,6 +877,18 @@ def attrs_configurations(self, session, object_ids, interest_attributes):
output[obj_id][attr["key"]] = attr["id"]
return output, hiearchical
+ def get_hierarchical_configurations(self, session, interest_attributes):
+ hier_attr_query = (
+ "select id, key, object_type_id, is_hierarchical, default"
+ " from CustomAttributeConfiguration"
+ " where key in ({}) and is_hierarchical is true"
+ )
+ if not interest_attributes:
+ return []
+ return list(session.query(hier_attr_query.format(
+ self.join_query_keys(interest_attributes),
+ )).all())
+
def register(session):
PushFrameValuesToTaskEvent(session).register()
diff --git a/openpype/modules/default_modules/ftrack/event_handlers_server/event_radio_buttons.py b/openpype/modules/ftrack/event_handlers_server/event_radio_buttons.py
similarity index 100%
rename from openpype/modules/default_modules/ftrack/event_handlers_server/event_radio_buttons.py
rename to openpype/modules/ftrack/event_handlers_server/event_radio_buttons.py
diff --git a/openpype/modules/default_modules/ftrack/event_handlers_server/event_sync_links.py b/openpype/modules/ftrack/event_handlers_server/event_sync_links.py
similarity index 98%
rename from openpype/modules/default_modules/ftrack/event_handlers_server/event_sync_links.py
rename to openpype/modules/ftrack/event_handlers_server/event_sync_links.py
index 83132acd85b..ae70c6756f2 100644
--- a/openpype/modules/default_modules/ftrack/event_handlers_server/event_sync_links.py
+++ b/openpype/modules/ftrack/event_handlers_server/event_sync_links.py
@@ -1,7 +1,7 @@
from pymongo import UpdateOne
from bson.objectid import ObjectId
-from avalon.api import AvalonMongoDB
+from openpype.pipeline import AvalonMongoDB
from openpype_modules.ftrack.lib import (
CUST_ATTR_ID_KEY,
@@ -128,7 +128,7 @@ def _update_in_links(self, session, ftrack_ids, project_id):
def _get_mongo_ids_by_ftrack_ids(self, session, attr_id, ftrack_ids):
output = query_custom_attributes(
- session, [attr_id], ftrack_ids
+ session, [attr_id], ftrack_ids, True
)
mongo_id_by_ftrack_id = {}
for item in output:
diff --git a/openpype/modules/default_modules/ftrack/event_handlers_server/event_sync_to_avalon.py b/openpype/modules/ftrack/event_handlers_server/event_sync_to_avalon.py
similarity index 91%
rename from openpype/modules/default_modules/ftrack/event_handlers_server/event_sync_to_avalon.py
rename to openpype/modules/ftrack/event_handlers_server/event_sync_to_avalon.py
index a4982627ff3..e549de7ed09 100644
--- a/openpype/modules/default_modules/ftrack/event_handlers_server/event_sync_to_avalon.py
+++ b/openpype/modules/ftrack/event_handlers_server/event_sync_to_avalon.py
@@ -12,19 +12,30 @@
import arrow
import ftrack_api
-from avalon import schema
-from avalon.api import AvalonMongoDB
+from openpype.client import (
+ get_project,
+ get_assets,
+ get_archived_assets,
+ get_asset_ids_with_subsets
+)
+from openpype.client.operations import CURRENT_ASSET_DOC_SCHEMA
+from openpype.pipeline import AvalonMongoDB, schema
from openpype_modules.ftrack.lib import (
get_openpype_attr,
+ query_custom_attributes,
CUST_ATTR_ID_KEY,
CUST_ATTR_AUTO_SYNC,
+ FPS_KEYS,
avalon_sync,
BaseEvent
)
-from openpype.lib import CURRENT_DOC_SCHEMAS
+from openpype_modules.ftrack.lib.avalon_sync import (
+ convert_to_fps,
+ InvalidFpsValue
+)
class SyncToAvalonEvent(BaseEvent):
@@ -48,8 +59,8 @@ class SyncToAvalonEvent(BaseEvent):
)
entities_query_by_id = (
- "select id, name, parent_id, link, custom_attributes from TypedContext"
- " where project_id is \"{}\" and id in ({})"
+ "select id, name, parent_id, link, custom_attributes, description"
+ " from TypedContext where project_id is \"{}\" and id in ({})"
)
# useful for getting all tasks for asset
@@ -144,12 +155,11 @@ def cust_attr_types_by_id(self):
@property
def avalon_entities(self):
if self._avalon_ents is None:
+ project_name = self.cur_project["full_name"]
self.dbcon.install()
- self.dbcon.Session["AVALON_PROJECT"] = (
- self.cur_project["full_name"]
- )
- avalon_project = self.dbcon.find_one({"type": "project"})
- avalon_entities = list(self.dbcon.find({"type": "asset"}))
+ self.dbcon.Session["AVALON_PROJECT"] = project_name
+ avalon_project = get_project(project_name)
+ avalon_entities = list(get_assets(project_name))
self._avalon_ents = (avalon_project, avalon_entities)
return self._avalon_ents
@@ -193,8 +203,10 @@ def avalon_ents_by_ftrack_id(self):
if proj:
ftrack_id = proj["data"].get("ftrackId")
if ftrack_id is None:
- ftrack_id = self._update_project_ftrack_id()
- proj["data"]["ftrackId"] = ftrack_id
+ self.handle_missing_ftrack_id(proj)
+ ftrack_id = proj["data"]["ftrackId"]
+ self._avalon_ents_by_ftrack_id[ftrack_id] = proj
+
self._avalon_ents_by_ftrack_id[ftrack_id] = proj
for ent in ents:
ftrack_id = ent["data"].get("ftrackId")
@@ -203,39 +215,95 @@ def avalon_ents_by_ftrack_id(self):
self._avalon_ents_by_ftrack_id[ftrack_id] = ent
return self._avalon_ents_by_ftrack_id
- def _update_project_ftrack_id(self):
- ftrack_id = self.cur_project["id"]
+ def handle_missing_ftrack_id(self, doc):
+ # TODO handling of missing ftrack id is primarily issue of editorial
+ # publishing it would be better to find out what causes that
+ # ftrack id is removed during the publishing
+ ftrack_id = doc["data"].get("ftrackId")
+ if ftrack_id is not None:
+ return
+
+ if doc["type"] == "project":
+ ftrack_id = self.cur_project["id"]
+ self.dbcon.update_one(
+ {"type": "project"},
+ {"$set": {
+ "data.ftrackId": ftrack_id,
+ "data.entityType": self.cur_project.entity_type
+ }}
+ )
+
+ doc["data"]["ftrackId"] = ftrack_id
+ doc["data"]["entityType"] = self.cur_project.entity_type
+ self.log.info("Updated ftrack id of project \"{}\"".format(
+ self.cur_project["full_name"]
+ ))
+ return
+
+ if doc["type"] != "asset":
+ return
+
+ doc_parents = doc.get("data", {}).get("parents")
+ if doc_parents is None:
+ return
+
+ entities = self.process_session.query((
+ "select id, link from TypedContext"
+ " where project_id is \"{}\" and name is \"{}\""
+ ).format(self.cur_project["id"], doc["name"])).all()
+ self.log.info("Entities: {}".format(str(entities)))
+ matching_entity = None
+ for entity in entities:
+ parents = []
+ for item in entity["link"]:
+ if item["id"] == entity["id"]:
+ break
+ low_type = item["type"].lower()
+ if low_type == "typedcontext":
+ parents.append(item["name"])
+ if doc_parents == parents:
+ matching_entity = entity
+ break
+
+ if matching_entity is None:
+ return
+
+ ftrack_id = matching_entity["id"]
self.dbcon.update_one(
- {"type": "project"},
- {"$set": {"data.ftrackId": ftrack_id}}
+ {"_id": doc["_id"]},
+ {"$set": {
+ "data.ftrackId": ftrack_id,
+ "data.entityType": matching_entity.entity_type
+ }}
)
-
- return ftrack_id
+ doc["data"]["ftrackId"] = ftrack_id
+ doc["data"]["entityType"] = matching_entity.entity_type
+
+ entity_path_items = []
+ for item in entity["link"]:
+ entity_path_items.append(item["name"])
+ self.log.info("Updated ftrack id of entity \"{}\"".format(
+ "/".join(entity_path_items)
+ ))
+ self._avalon_ents_by_ftrack_id[ftrack_id] = doc
@property
- def avalon_subsets_by_parents(self):
- if self._avalon_subsets_by_parents is None:
- self._avalon_subsets_by_parents = collections.defaultdict(list)
- self.dbcon.install()
- self.dbcon.Session["AVALON_PROJECT"] = (
- self.cur_project["full_name"]
+ def avalon_asset_ids_with_subsets(self):
+ if self._avalon_asset_ids_with_subsets is None:
+ project_name = self.cur_project["full_name"]
+ self._avalon_asset_ids_with_subsets = get_asset_ids_with_subsets(
+ project_name
)
- for subset in self.dbcon.find({"type": "subset"}):
- self._avalon_subsets_by_parents[subset["parent"]].append(
- subset
- )
- return self._avalon_subsets_by_parents
+
+ return self._avalon_asset_ids_with_subsets
@property
def avalon_archived_by_id(self):
if self._avalon_archived_by_id is None:
self._avalon_archived_by_id = {}
- self.dbcon.install()
- self.dbcon.Session["AVALON_PROJECT"] = (
- self.cur_project["full_name"]
- )
- for asset in self.dbcon.find({"type": "archived_asset"}):
+ project_name = self.cur_project["full_name"]
+ for asset in get_archived_assets(project_name):
self._avalon_archived_by_id[asset["_id"]] = asset
return self._avalon_archived_by_id
@@ -257,7 +325,7 @@ def changeability_by_mongo_id(self):
avalon_project, avalon_entities = self.avalon_entities
self._changeability_by_mongo_id[avalon_project["_id"]] = False
self._bubble_changeability(
- list(self.avalon_subsets_by_parents.keys())
+ list(self.avalon_asset_ids_with_subsets)
)
return self._changeability_by_mongo_id
@@ -379,14 +447,9 @@ def _bubble_changeability(self, unchangeable_ids):
if not entity:
# if entity is not found then it is subset without parent
if entity_id in unchangeable_ids:
- _subset_ids = [
- str(sub["_id"]) for sub in
- self.avalon_subsets_by_parents[entity_id]
- ]
- joined_subset_ids = "| ".join(_subset_ids)
self.log.warning((
- "Parent <{}> for subsets <{}> does not exist"
- ).format(str(entity_id), joined_subset_ids))
+ "Parent <{}> with subsets does not exist"
+ ).format(str(entity_id)))
else:
self.log.warning((
"In avalon are entities without valid parents that"
@@ -413,7 +476,7 @@ def reset_variables(self):
self._avalon_ents_by_parent_id = None
self._avalon_ents_by_ftrack_id = None
self._avalon_ents_by_name = None
- self._avalon_subsets_by_parents = None
+ self._avalon_asset_ids_with_subsets = None
self._changeability_by_mongo_id = None
self._avalon_archived_by_id = None
self._avalon_archived_by_name = None
@@ -634,13 +697,22 @@ def launch(self, session, event):
continue
auto_sync = changes[CUST_ATTR_AUTO_SYNC]["new"]
- if auto_sync == "1":
+ turned_on = auto_sync == "1"
+ ft_project = self.cur_project
+ username = self._get_username(session, event)
+ message = (
+ "Auto sync was turned {} for project \"{}\" by \"{}\"."
+ ).format(
+ "on" if turned_on else "off",
+ ft_project["full_name"],
+ username
+ )
+ if turned_on:
+ message += " Triggering syncToAvalon action."
+ self.log.debug(message)
+
+ if turned_on:
# Trigger sync to avalon action if auto sync was turned on
- ft_project = self.cur_project
- self.log.debug((
- "Auto sync was turned on for project <{}>."
- " Triggering syncToAvalon action."
- ).format(ft_project["full_name"]))
selection = [{
"entityId": ft_project["id"],
"entityType": "show"
@@ -788,6 +860,26 @@ def launch(self, session, event):
self.report()
return True
+ def _get_username(self, session, event):
+ username = "Unknown"
+ event_source = event.get("source")
+ if not event_source:
+ return username
+ user_info = event_source.get("user")
+ if not user_info:
+ return username
+ user_id = user_info.get("id")
+ if not user_id:
+ return username
+
+ user_entity = session.query(
+ "User where id is {}".format(user_id)
+ ).first()
+ if user_entity:
+ username = user_entity["username"] or username
+ return username
+
+
def process_removed(self):
"""
Handles removed entities (not removed tasks - handle separately).
@@ -851,7 +943,14 @@ def process_removed(self):
if vis_par is None:
vis_par = proj["_id"]
parent_ent = self.avalon_ents_by_id[vis_par]
- parent_ftrack_id = parent_ent["data"]["ftrackId"]
+
+ parent_ftrack_id = parent_ent["data"].get("ftrackId")
+ if parent_ftrack_id is None:
+ self.handle_missing_ftrack_id(parent_ent)
+ parent_ftrack_id = parent_ent["data"].get("ftrackId")
+ if parent_ftrack_id is None:
+ continue
+
parent_ftrack_ent = self.ftrack_ents_by_id.get(
parent_ftrack_id
)
@@ -1073,9 +1172,8 @@ def check_names_synchronizable(self, names):
self.create_entity_in_avalon(entity, parent_avalon_ent)
for child in entity["children"]:
- if child.entity_type.lower() == "task":
- continue
- children_queue.append(child)
+ if child.entity_type.lower() != "task":
+ children_queue.append(child)
while children_queue:
entity = children_queue.popleft()
@@ -1138,22 +1236,42 @@ def create_entity_in_avalon(self, ftrack_ent, parent_avalon):
"_id": mongo_id,
"name": name,
"type": "asset",
- "schema": CURRENT_DOC_SCHEMAS["asset"],
+ "schema": CURRENT_ASSET_DOC_SCHEMA,
"parent": proj["_id"],
"data": {
"ftrackId": ftrack_ent["id"],
"entityType": ftrack_ent.entity_type,
"parents": parents,
"tasks": {},
- "visualParent": vis_par
+ "visualParent": vis_par,
+ "description": ftrack_ent["description"]
}
}
+ invalid_fps_items = []
cust_attrs = self.get_cust_attr_values(ftrack_ent)
for key, val in cust_attrs.items():
if key.startswith("avalon_"):
continue
+
+ if key in FPS_KEYS:
+ try:
+ val = convert_to_fps(val)
+ except InvalidFpsValue:
+ invalid_fps_items.append((ftrack_ent["id"], val))
+ continue
+
final_entity["data"][key] = val
+ if invalid_fps_items:
+ fps_msg = (
+ "These entities have invalid fps value in custom attributes"
+ )
+ items = []
+ for entity_id, value in invalid_fps_items:
+ ent_path = self.get_ent_path(entity_id)
+ items.append("{} - \"{}\"".format(ent_path, value))
+ self.report_items["error"][fps_msg] = items
+
_mongo_id_str = cust_attrs.get(CUST_ATTR_ID_KEY)
if _mongo_id_str:
try:
@@ -1822,7 +1940,15 @@ def process_updated(self):
if ent_cust_attrs is None:
ent_cust_attrs = {}
- for key, values in ent_info["changes"].items():
+ ent_changes = ent_info["changes"]
+ if "description" in ent_changes:
+ if "data" not in self.updates[mongo_id]:
+ self.updates[mongo_id]["data"] = {}
+ self.updates[mongo_id]["data"]["description"] = (
+ ent_changes["description"]["new"] or ""
+ )
+
+ for key, values in ent_changes.items():
if key in hier_attrs_by_key:
self.hier_cust_attrs_changes[key].append(ftrack_id)
continue
@@ -2095,7 +2221,13 @@ def process_hier_cleanup(self):
vis_par = avalon_ent["parent"]
parent_ent = self.avalon_ents_by_id[vis_par]
- parent_ftrack_id = parent_ent["data"]["ftrackId"]
+ parent_ftrack_id = parent_ent["data"].get("ftrackId")
+ if parent_ftrack_id is None:
+ self.handle_missing_ftrack_id(parent_ent)
+ parent_ftrack_id = parent_ent["data"].get("ftrackId")
+ if parent_ftrack_id is None:
+ continue
+
if parent_ftrack_id not in entities_dict:
entities_dict[parent_ftrack_id] = {
"children": [],
@@ -2122,22 +2254,12 @@ def process_hier_cleanup(self):
for key in hier_cust_attrs_keys:
configuration_ids.add(hier_attr_id_by_key[key])
- entity_ids_joined = self.join_query_keys(cust_attrs_ftrack_ids)
- attributes_joined = self.join_query_keys(configuration_ids)
-
- queries = [{
- "action": "query",
- "expression": (
- "select value, entity_id, configuration_id"
- " from CustomAttributeValue "
- "where entity_id in ({}) and configuration_id in ({})"
- ).format(entity_ids_joined, attributes_joined)
- }]
-
- if hasattr(self.process_session, "call"):
- [values] = self.process_session.call(queries)
- else:
- [values] = self.process_session._call(queries)
+ values = query_custom_attributes(
+ self.process_session,
+ configuration_ids,
+ cust_attrs_ftrack_ids,
+ True
+ )
ftrack_project_id = self.cur_project["id"]
@@ -2156,13 +2278,21 @@ def process_hier_cleanup(self):
)
convert_types_by_id[attr_id] = convert_type
+ default_value = attr["default"]
+ if key in FPS_KEYS:
+ try:
+ default_value = convert_to_fps(default_value)
+ except InvalidFpsValue:
+ pass
+
entities_dict[ftrack_project_id]["hier_attrs"][key] = (
attr["default"]
)
# PREPARE DATA BEFORE THIS
+ invalid_fps_items = []
avalon_hier = []
- for item in values["data"]:
+ for item in values:
value = item["value"]
if value is None:
continue
@@ -2174,8 +2304,25 @@ def process_hier_cleanup(self):
if convert_type:
value = convert_type(value)
+
+ if key in FPS_KEYS:
+ try:
+ value = convert_to_fps(value)
+ except InvalidFpsValue:
+ invalid_fps_items.append((entity_id, value))
+ continue
entities_dict[entity_id]["hier_attrs"][key] = value
+ if invalid_fps_items:
+ fps_msg = (
+ "These entities have invalid fps value in custom attributes"
+ )
+ items = []
+ for entity_id, value in invalid_fps_items:
+ ent_path = self.get_ent_path(entity_id)
+ items.append("{} - \"{}\"".format(ent_path, value))
+ self.report_items["error"][fps_msg] = items
+
# Get dictionary with not None hierarchical values to pull to childs
project_values = {}
for key, value in (
diff --git a/openpype/modules/default_modules/ftrack/event_handlers_server/event_task_to_parent_status.py b/openpype/modules/ftrack/event_handlers_server/event_task_to_parent_status.py
similarity index 100%
rename from openpype/modules/default_modules/ftrack/event_handlers_server/event_task_to_parent_status.py
rename to openpype/modules/ftrack/event_handlers_server/event_task_to_parent_status.py
diff --git a/openpype/modules/default_modules/ftrack/event_handlers_server/event_task_to_version_status.py b/openpype/modules/ftrack/event_handlers_server/event_task_to_version_status.py
similarity index 100%
rename from openpype/modules/default_modules/ftrack/event_handlers_server/event_task_to_version_status.py
rename to openpype/modules/ftrack/event_handlers_server/event_task_to_version_status.py
diff --git a/openpype/modules/default_modules/ftrack/event_handlers_server/event_thumbnail_updates.py b/openpype/modules/ftrack/event_handlers_server/event_thumbnail_updates.py
similarity index 100%
rename from openpype/modules/default_modules/ftrack/event_handlers_server/event_thumbnail_updates.py
rename to openpype/modules/ftrack/event_handlers_server/event_thumbnail_updates.py
diff --git a/openpype/modules/default_modules/ftrack/event_handlers_server/event_user_assigment.py b/openpype/modules/ftrack/event_handlers_server/event_user_assigment.py
similarity index 91%
rename from openpype/modules/default_modules/ftrack/event_handlers_server/event_user_assigment.py
rename to openpype/modules/ftrack/event_handlers_server/event_user_assigment.py
index efc1e767755..88d252e8cf8 100644
--- a/openpype/modules/default_modules/ftrack/event_handlers_server/event_user_assigment.py
+++ b/openpype/modules/ftrack/event_handlers_server/event_user_assigment.py
@@ -1,14 +1,11 @@
-import os
import re
import subprocess
+from openpype.client import get_asset_by_id, get_asset_by_name
+from openpype.settings import get_project_settings
+from openpype.pipeline import Anatomy
from openpype_modules.ftrack.lib import BaseEvent
from openpype_modules.ftrack.lib.avalon_sync import CUST_ATTR_ID_KEY
-from avalon.api import AvalonMongoDB
-
-from bson.objectid import ObjectId
-
-from openpype.api import Anatomy, get_project_settings
class UserAssigmentEvent(BaseEvent):
@@ -37,8 +34,6 @@ class UserAssigmentEvent(BaseEvent):
3) path to publish files of task user was (de)assigned to
"""
- db_con = AvalonMongoDB()
-
def error(self, *err):
for e in err:
self.log.error(e)
@@ -87,8 +82,8 @@ def _get_task_and_user(self, session, action, changes):
if not user_id:
return None, None
- task = session.query('Task where id is "{}"'.format(task_id)).one()
- user = session.query('User where id is "{}"'.format(user_id)).one()
+ task = session.query('Task where id is "{}"'.format(task_id)).first()
+ user = session.query('User where id is "{}"'.format(user_id)).first()
return task, user
@@ -102,26 +97,16 @@ def _get_asset(self, task):
:rtype: dict
"""
parent = task['parent']
- self.db_con.install()
- self.db_con.Session['AVALON_PROJECT'] = task['project']['full_name']
-
+ project_name = task["project"]["full_name"]
avalon_entity = None
parent_id = parent['custom_attributes'].get(CUST_ATTR_ID_KEY)
if parent_id:
- parent_id = ObjectId(parent_id)
- avalon_entity = self.db_con.find_one({
- '_id': parent_id,
- 'type': 'asset'
- })
+ avalon_entity = get_asset_by_id(project_name, parent_id)
if not avalon_entity:
- avalon_entity = self.db_con.find_one({
- 'type': 'asset',
- 'name': parent['name']
- })
+ avalon_entity = get_asset_by_name(project_name, parent["name"])
if not avalon_entity:
- self.db_con.uninstall()
msg = 'Entity "{}" not found in avalon database'.format(
parent['name']
)
@@ -130,7 +115,6 @@ def _get_asset(self, task):
'success': False,
'message': msg
}
- self.db_con.uninstall()
return avalon_entity
def _get_hierarchy(self, asset):
diff --git a/openpype/modules/default_modules/ftrack/event_handlers_server/event_version_to_task_statuses.py b/openpype/modules/ftrack/event_handlers_server/event_version_to_task_statuses.py
similarity index 100%
rename from openpype/modules/default_modules/ftrack/event_handlers_server/event_version_to_task_statuses.py
rename to openpype/modules/ftrack/event_handlers_server/event_version_to_task_statuses.py
diff --git a/openpype/modules/default_modules/ftrack/event_handlers_user/action_applications.py b/openpype/modules/ftrack/event_handlers_user/action_applications.py
similarity index 94%
rename from openpype/modules/default_modules/ftrack/event_handlers_user/action_applications.py
rename to openpype/modules/ftrack/event_handlers_user/action_applications.py
index 6d45d439586..102f04c9566 100644
--- a/openpype/modules/default_modules/ftrack/event_handlers_user/action_applications.py
+++ b/openpype/modules/ftrack/event_handlers_user/action_applications.py
@@ -1,13 +1,13 @@
import os
-from uuid import uuid4
+from openpype.client import get_project
from openpype_modules.ftrack.lib import BaseAction
-from openpype.lib import (
+from openpype.lib.applications import (
ApplicationManager,
ApplicationLaunchFailed,
- ApplictionExecutableNotFound
+ ApplictionExecutableNotFound,
+ CUSTOM_LAUNCH_APP_GROUPS
)
-from avalon.api import AvalonMongoDB
class AppplicationsAction(BaseAction):
@@ -25,7 +25,6 @@ def __init__(self, *args, **kwargs):
super(AppplicationsAction, self).__init__(*args, **kwargs)
self.application_manager = ApplicationManager()
- self.dbcon = AvalonMongoDB()
@property
def discover_identifier(self):
@@ -110,12 +109,7 @@ def discover(self, session, entities, event):
if avalon_project_doc is None:
ft_project = self.get_project_from_entity(entity)
project_name = ft_project["full_name"]
- if not self.dbcon.is_installed():
- self.dbcon.install()
- self.dbcon.Session["AVALON_PROJECT"] = project_name
- avalon_project_doc = self.dbcon.find_one({
- "type": "project"
- }) or False
+ avalon_project_doc = get_project(project_name) or False
event["data"]["avalon_project_doc"] = avalon_project_doc
if not avalon_project_doc:
@@ -136,6 +130,9 @@ def discover(self, session, entities, event):
if not app or not app.enabled:
continue
+ if app.group.name in CUSTOM_LAUNCH_APP_GROUPS:
+ continue
+
app_icon = app.icon
if app_icon and self.icon_url:
try:
diff --git a/openpype/modules/default_modules/ftrack/event_handlers_user/action_batch_task_creation.py b/openpype/modules/ftrack/event_handlers_user/action_batch_task_creation.py
similarity index 100%
rename from openpype/modules/default_modules/ftrack/event_handlers_user/action_batch_task_creation.py
rename to openpype/modules/ftrack/event_handlers_user/action_batch_task_creation.py
diff --git a/openpype/modules/default_modules/ftrack/event_handlers_user/action_clean_hierarchical_attributes.py b/openpype/modules/ftrack/event_handlers_user/action_clean_hierarchical_attributes.py
similarity index 85%
rename from openpype/modules/default_modules/ftrack/event_handlers_user/action_clean_hierarchical_attributes.py
rename to openpype/modules/ftrack/event_handlers_user/action_clean_hierarchical_attributes.py
index dc97ed972d8..f06162bfda7 100644
--- a/openpype/modules/default_modules/ftrack/event_handlers_user/action_clean_hierarchical_attributes.py
+++ b/openpype/modules/ftrack/event_handlers_user/action_clean_hierarchical_attributes.py
@@ -19,8 +19,8 @@ class CleanHierarchicalAttrsAction(BaseAction):
" from TypedContext where project_id is \"{}\""
)
cust_attr_query = (
- "select value, entity_id from CustomAttributeValue "
- "where entity_id in ({}) and configuration_id is \"{}\""
+ "select value, entity_id from CustomAttributeValue"
+ " where entity_id in ({}) and configuration_id is \"{}\""
)
settings_key = "clean_hierarchical_attr"
@@ -65,17 +65,14 @@ def launch(self, session, entities, event):
)
)
configuration_id = attr["id"]
- call_expr = [{
- "action": "query",
- "expression": self.cust_attr_query.format(
+ values = session.query(
+ self.cust_attr_query.format(
entity_ids_joined, configuration_id
)
- }]
-
- [values] = self.session.call(call_expr)
+ ).all()
data = {}
- for item in values["data"]:
+ for item in values:
value = item["value"]
if value is None:
data[item["entity_id"]] = value
@@ -90,10 +87,10 @@ def launch(self, session, entities, event):
len(data), configuration_key
))
for entity_id, value in data.items():
- entity_key = collections.OrderedDict({
- "configuration_id": configuration_id,
- "entity_id": entity_id
- })
+ entity_key = collections.OrderedDict((
+ ("configuration_id", configuration_id),
+ ("entity_id", entity_id)
+ ))
session.recorded_operations.push(
ftrack_api.operation.DeleteEntityOperation(
"CustomAttributeValue",
diff --git a/openpype/modules/default_modules/ftrack/event_handlers_user/action_client_review_sort.py b/openpype/modules/ftrack/event_handlers_user/action_client_review_sort.py
similarity index 100%
rename from openpype/modules/default_modules/ftrack/event_handlers_user/action_client_review_sort.py
rename to openpype/modules/ftrack/event_handlers_user/action_client_review_sort.py
diff --git a/openpype/modules/default_modules/ftrack/event_handlers_user/action_component_open.py b/openpype/modules/ftrack/event_handlers_user/action_component_open.py
similarity index 100%
rename from openpype/modules/default_modules/ftrack/event_handlers_user/action_component_open.py
rename to openpype/modules/ftrack/event_handlers_user/action_component_open.py
diff --git a/openpype/modules/default_modules/ftrack/event_handlers_user/action_create_cust_attrs.py b/openpype/modules/ftrack/event_handlers_user/action_create_cust_attrs.py
similarity index 95%
rename from openpype/modules/default_modules/ftrack/event_handlers_user/action_create_cust_attrs.py
rename to openpype/modules/ftrack/event_handlers_user/action_create_cust_attrs.py
index 0bd243ab4cd..d04440a5648 100644
--- a/openpype/modules/default_modules/ftrack/event_handlers_user/action_create_cust_attrs.py
+++ b/openpype/modules/ftrack/event_handlers_user/action_create_cust_attrs.py
@@ -11,6 +11,7 @@
CUST_ATTR_TOOLS,
CUST_ATTR_APPLICATIONS,
CUST_ATTR_INTENT,
+ FPS_KEYS,
default_custom_attributes_definition,
app_definitions_from_app_manager,
@@ -139,9 +140,9 @@ class CustomAttributes(BaseAction):
identifier = 'create.update.attributes'
#: Action label.
label = "OpenPype Admin"
- variant = '- Create/Update Avalon Attributes'
+ variant = '- Create/Update Custom Attributes'
#: Action description.
- description = 'Creates Avalon/Mongo ID for double check'
+ description = 'Creates required custom attributes in ftrack'
icon = statics_icon("ftrack", "action_icons", "OpenPypeAdmin.svg")
settings_key = "create_update_attributes"
@@ -306,8 +307,8 @@ def convert_mongo_id_to_hierarchical(
}
cust_attr_query = (
- "select value, entity_id from ContextCustomAttributeValue "
- "where configuration_id is {}"
+ "select value, entity_id from CustomAttributeValue"
+ " where configuration_id is {}"
)
for attr_def in object_type_attrs:
attr_ent_type = attr_def["entity_type"]
@@ -328,21 +329,14 @@ def convert_mongo_id_to_hierarchical(
self.log.debug((
"Converting Avalon MongoID attr for Entity type \"{}\"."
).format(entity_type_label))
-
- call_expr = [{
- "action": "query",
- "expression": cust_attr_query.format(attr_def["id"])
- }]
- if hasattr(session, "call"):
- [values] = session.call(call_expr)
- else:
- [values] = session._call(call_expr)
-
- for value in values["data"]:
- table_values = collections.OrderedDict({
- "configuration_id": hierarchical_attr["id"],
- "entity_id": value["entity_id"]
- })
+ values = session.query(
+ cust_attr_query.format(attr_def["id"])
+ ).all()
+ for value in values:
+ table_values = collections.OrderedDict([
+ ("configuration_id", hierarchical_attr["id"]),
+ ("entity_id", value["entity_id"])
+ ])
session.recorded_operations.push(
ftrack_api.operation.UpdateEntityOperation(
@@ -526,20 +520,28 @@ def process_attr_data(self, cust_attr_data, event):
self.show_message(event, msg)
def process_attribute(self, data):
- existing_attrs = self.session.query(
- "CustomAttributeConfiguration"
- ).all()
+ existing_attrs = self.session.query((
+ "select is_hierarchical, key, type, entity_type, object_type_id"
+ " from CustomAttributeConfiguration"
+ )).all()
matching = []
+ is_hierarchical = data.get("is_hierarchical", False)
for attr in existing_attrs:
if (
- attr["key"] != data["key"] or
- attr["type"]["name"] != data["type"]["name"]
+ is_hierarchical != attr["is_hierarchical"]
+ or attr["key"] != data["key"]
):
continue
- if data.get("is_hierarchical") is True:
- if attr["is_hierarchical"] is True:
- matching.append(attr)
+ if attr["type"]["name"] != data["type"]["name"]:
+ if data["key"] in FPS_KEYS and attr["type"]["name"] == "text":
+ self.log.info("Kept 'fps' as text custom attribute.")
+ return
+ continue
+
+ if is_hierarchical:
+ matching.append(attr)
+
elif "object_type_id" in data:
if (
attr["entity_type"] == data["entity_type"] and
diff --git a/openpype/modules/ftrack/event_handlers_user/action_create_folders.py b/openpype/modules/ftrack/event_handlers_user/action_create_folders.py
new file mode 100644
index 00000000000..9806f83773d
--- /dev/null
+++ b/openpype/modules/ftrack/event_handlers_user/action_create_folders.py
@@ -0,0 +1,283 @@
+import os
+import collections
+import copy
+from openpype.pipeline import Anatomy
+from openpype_modules.ftrack.lib import BaseAction, statics_icon
+
+
+class CreateFolders(BaseAction):
+ identifier = "create.folders"
+ label = "Create Folders"
+ icon = statics_icon("ftrack", "action_icons", "CreateFolders.svg")
+
+ def discover(self, session, entities, event):
+ for entity_item in event["data"]["selection"]:
+ if entity_item.get("entityType").lower() in ("task", "show"):
+ return True
+ return False
+
+ def interface(self, session, entities, event):
+ if event["data"].get("values", {}):
+ return
+
+ with_interface = False
+ for entity in entities:
+ if entity.entity_type.lower() != "task":
+ with_interface = True
+ break
+
+ if "values" not in event["data"]:
+ event["data"]["values"] = {}
+
+ event["data"]["values"]["with_interface"] = with_interface
+ if not with_interface:
+ return
+
+ title = "Create folders"
+
+ entity_name = entity["name"]
+ msg = (
+ "Do you want create folders also"
+ " for all children of your selection? "
+ )
+ if entity.entity_type.lower() == "project":
+ entity_name = entity["full_name"]
+ msg = msg.replace(" also", "")
+ msg += "(Project root won't be created if not checked) "
+ items = [
+ {
+ "type": "label",
+ "value": msg.format(entity_name)
+ },
+ {
+ "type": "label",
+ "value": "With all chilren entities"
+ },
+ {
+ "name": "children_included",
+ "type": "boolean",
+ "value": False
+ },
+ {
+ "type": "hidden",
+ "name": "with_interface",
+ "value": with_interface
+ }
+ ]
+
+ return {
+ "items": items,
+ "title": title
+ }
+
+ def launch(self, session, entities, event):
+ '''Callback method for custom action.'''
+
+ if "values" not in event["data"]:
+ return
+
+ with_interface = event["data"]["values"]["with_interface"]
+ with_childrens = True
+ if with_interface:
+ with_childrens = event["data"]["values"]["children_included"]
+
+ filtered_entities = []
+ for entity in entities:
+ low_context_type = entity["context_type"].lower()
+ if low_context_type in ("task", "show"):
+ if not with_childrens and low_context_type == "show":
+ continue
+ filtered_entities.append(entity)
+
+ if not filtered_entities:
+ return {
+ "success": True,
+ "message": 'Nothing was created'
+ }
+
+ project_entity = self.get_project_from_entity(filtered_entities[0])
+
+ project_name = project_entity["full_name"]
+ project_code = project_entity["name"]
+
+ task_entities = []
+ other_entities = []
+ self.get_all_entities(
+ session, entities, task_entities, other_entities
+ )
+ hierarchy = self.get_entities_hierarchy(
+ session, task_entities, other_entities
+ )
+ task_types = session.query("select id, name from Type").all()
+ task_type_names_by_id = {
+ task_type["id"]: task_type["name"]
+ for task_type in task_types
+ }
+
+ anatomy = Anatomy(project_name)
+
+ work_keys = ["work", "folder"]
+ work_template = anatomy.templates
+ for key in work_keys:
+ work_template = work_template[key]
+
+ publish_keys = ["publish", "folder"]
+ publish_template = anatomy.templates
+ for key in publish_keys:
+ publish_template = publish_template[key]
+
+ project_data = {
+ "project": {
+ "name": project_name,
+ "code": project_code
+ }
+ }
+
+ collected_paths = []
+ for item in hierarchy:
+ parent_entity, task_entities = item
+
+ parent_data = copy.deepcopy(project_data)
+
+ parents = parent_entity["link"][1:-1]
+ hierarchy_names = [p["name"] for p in parents]
+ hierarchy = "/".join(hierarchy_names)
+
+ if hierarchy_names:
+ parent_name = hierarchy_names[-1]
+ else:
+ parent_name = project_name
+
+ parent_data.update({
+ "asset": parent_entity["name"],
+ "hierarchy": hierarchy,
+ "parent": parent_name
+ })
+
+ if not task_entities:
+ # create path for entity
+ collected_paths.append(self.compute_template(
+ anatomy, parent_data, work_keys
+ ))
+ collected_paths.append(self.compute_template(
+ anatomy, parent_data, publish_keys
+ ))
+ continue
+
+ for task_entity in task_entities:
+ task_type_id = task_entity["type_id"]
+ task_type_name = task_type_names_by_id[task_type_id]
+ task_data = copy.deepcopy(parent_data)
+ task_data["task"] = {
+ "name": task_entity["name"],
+ "type": task_type_name
+ }
+
+ # Template wok
+ collected_paths.append(self.compute_template(
+ anatomy, task_data, work_keys
+ ))
+
+ # Template publish
+ collected_paths.append(self.compute_template(
+ anatomy, task_data, publish_keys
+ ))
+
+ if len(collected_paths) == 0:
+ return {
+ "success": True,
+ "message": "No project folders to create."
+ }
+
+ self.log.info("Creating folders:")
+
+ for path in set(collected_paths):
+ self.log.info(path)
+ if not os.path.exists(path):
+ os.makedirs(path)
+
+ return {
+ "success": True,
+ "message": "Successfully created project folders."
+ }
+
+ def get_all_entities(
+ self, session, entities, task_entities, other_entities
+ ):
+ if not entities:
+ return
+
+ no_task_entities = []
+ for entity in entities:
+ if entity.entity_type.lower() == "task":
+ task_entities.append(entity)
+ else:
+ no_task_entities.append(entity)
+
+ if not no_task_entities:
+ return task_entities
+
+ other_entities.extend(no_task_entities)
+
+ no_task_entity_ids = [entity["id"] for entity in no_task_entities]
+ next_entities = session.query((
+ "select id, parent_id"
+ " from TypedContext where parent_id in ({})"
+ ).format(self.join_query_keys(no_task_entity_ids))).all()
+
+ self.get_all_entities(
+ session, next_entities, task_entities, other_entities
+ )
+
+ def get_entities_hierarchy(self, session, task_entities, other_entities):
+ task_entity_ids = [entity["id"] for entity in task_entities]
+ full_task_entities = session.query((
+ "select id, name, type_id, parent_id"
+ " from TypedContext where id in ({})"
+ ).format(self.join_query_keys(task_entity_ids)))
+ task_entities_by_parent_id = collections.defaultdict(list)
+ for entity in full_task_entities:
+ parent_id = entity["parent_id"]
+ task_entities_by_parent_id[parent_id].append(entity)
+
+ output = []
+ if not task_entities_by_parent_id:
+ return output
+
+ other_ids = set()
+ for entity in other_entities:
+ other_ids.add(entity["id"])
+ other_ids |= set(task_entities_by_parent_id.keys())
+
+ parent_entities = session.query((
+ "select id, name from TypedContext where id in ({})"
+ ).format(self.join_query_keys(other_ids))).all()
+
+ for parent_entity in parent_entities:
+ parent_id = parent_entity["id"]
+ output.append((
+ parent_entity,
+ task_entities_by_parent_id[parent_id]
+ ))
+
+ return output
+
+ def compute_template(self, anatomy, data, anatomy_keys):
+ filled_template = anatomy.format_all(data)
+ for key in anatomy_keys:
+ filled_template = filled_template[key]
+
+ if filled_template.solved:
+ return os.path.normpath(filled_template)
+
+ self.log.warning(
+ "Template \"{}\" was not fully filled \"{}\"".format(
+ filled_template.template, filled_template
+ )
+ )
+ return os.path.normpath(filled_template.split("{")[0])
+
+
+def register(session):
+ """Register plugin. Called when used as an plugin."""
+ CreateFolders(session).register()
diff --git a/openpype/modules/default_modules/ftrack/event_handlers_user/action_create_project_structure.py b/openpype/modules/ftrack/event_handlers_user/action_create_project_structure.py
similarity index 95%
rename from openpype/modules/default_modules/ftrack/event_handlers_user/action_create_project_structure.py
rename to openpype/modules/ftrack/event_handlers_user/action_create_project_structure.py
index 94f359c317f..7c896570b1e 100644
--- a/openpype/modules/default_modules/ftrack/event_handlers_user/action_create_project_structure.py
+++ b/openpype/modules/ftrack/event_handlers_user/action_create_project_structure.py
@@ -1,9 +1,10 @@
-import os
import re
-import json
+from openpype.pipeline.project_folders import (
+ get_project_basic_paths,
+ create_project_folders,
+)
from openpype_modules.ftrack.lib import BaseAction, statics_icon
-from openpype.api import get_project_basic_paths, create_project_folders
class CreateProjectFolders(BaseAction):
@@ -83,9 +84,14 @@ def launch(self, session, entities, event):
}
# Invoking OpenPype API to create the project folders
- create_project_folders(basic_paths, project_name)
+ create_project_folders(project_name, basic_paths)
self.create_ftrack_entities(basic_paths, project_entity)
+ self.trigger_event(
+ "openpype.project.structure.created",
+ {"project_name": project_name}
+ )
+
except Exception as exc:
self.log.warning("Creating of structure crashed.", exc_info=True)
session.rollback()
diff --git a/openpype/modules/default_modules/ftrack/event_handlers_user/action_delete_asset.py b/openpype/modules/ftrack/event_handlers_user/action_delete_asset.py
similarity index 81%
rename from openpype/modules/default_modules/ftrack/event_handlers_user/action_delete_asset.py
rename to openpype/modules/ftrack/event_handlers_user/action_delete_asset.py
index d3cc0ad9714..03d029b0c18 100644
--- a/openpype/modules/default_modules/ftrack/event_handlers_user/action_delete_asset.py
+++ b/openpype/modules/ftrack/event_handlers_user/action_delete_asset.py
@@ -3,8 +3,11 @@
from datetime import datetime
from bson.objectid import ObjectId
+
+from openpype.client import get_assets, get_subsets
+from openpype.pipeline import AvalonMongoDB
from openpype_modules.ftrack.lib import BaseAction, statics_icon
-from avalon.api import AvalonMongoDB
+from openpype_modules.ftrack.lib.avalon_sync import create_chunks
class DeleteAssetSubset(BaseAction):
@@ -89,10 +92,8 @@ def interface(self, session, entities, event):
continue
ftrack_id = entity.get("entityId")
- if not ftrack_id:
- continue
-
- ftrack_ids.append(ftrack_id)
+ if ftrack_id:
+ ftrack_ids.append(ftrack_id)
if project_in_selection:
msg = "It is not possible to use this action on project entity."
@@ -118,69 +119,75 @@ def interface(self, session, entities, event):
"message": "Invalid selection for this action (Bug)"
}
- if entities[0].entity_type.lower() == "project":
- project = entities[0]
- else:
- project = entities[0]["project"]
-
+ project = self.get_project_from_entity(entities[0], session)
project_name = project["full_name"]
self.dbcon.Session["AVALON_PROJECT"] = project_name
- selected_av_entities = list(self.dbcon.find({
- "type": "asset",
- "data.ftrackId": {"$in": ftrack_ids}
- }))
+ asset_docs = list(get_assets(
+ project_name,
+ fields=["_id", "name", "data.ftrackId", "data.parents"]
+ ))
+ selected_av_entities = []
+ found_ftrack_ids = set()
+ asset_docs_by_name = collections.defaultdict(list)
+ for asset_doc in asset_docs:
+ ftrack_id = asset_doc["data"].get("ftrackId")
+ if ftrack_id:
+ found_ftrack_ids.add(ftrack_id)
+ if ftrack_id in entity_mapping:
+ selected_av_entities.append(asset_doc)
+
+ asset_name = asset_doc["name"]
+ asset_docs_by_name[asset_name].append(asset_doc)
+
found_without_ftrack_id = {}
- if len(selected_av_entities) != len(ftrack_ids):
- found_ftrack_ids = [
- ent["data"]["ftrackId"] for ent in selected_av_entities
- ]
- for ftrack_id, entity in entity_mapping.items():
- if ftrack_id in found_ftrack_ids:
- continue
+ for ftrack_id, entity in entity_mapping.items():
+ if ftrack_id in found_ftrack_ids:
+ continue
- av_ents_by_name = list(self.dbcon.find({
- "type": "asset",
- "name": entity["name"]
- }))
- if not av_ents_by_name:
- continue
+ av_ents_by_name = asset_docs_by_name[entity["name"]]
+ if not av_ents_by_name:
+ continue
- ent_path_items = [ent["name"] for ent in entity["link"]]
- parents = ent_path_items[1:len(ent_path_items)-1:]
- # TODO we should say to user that
- # few of them are missing in avalon
- for av_ent in av_ents_by_name:
- if av_ent["data"]["parents"] != parents:
- continue
+ ent_path_items = [ent["name"] for ent in entity["link"]]
+ end_index = len(ent_path_items) - 1
+ parents = ent_path_items[1:end_index:]
+ # TODO we should say to user that
+ # few of them are missing in avalon
+ for av_ent in av_ents_by_name:
+ if av_ent["data"]["parents"] != parents:
+ continue
- # TODO we should say to user that found entity
- # with same name does not match same ftrack id?
- if "ftrackId" not in av_ent["data"]:
- selected_av_entities.append(av_ent)
- found_without_ftrack_id[str(av_ent["_id"])] = ftrack_id
- break
+ # TODO we should say to user that found entity
+ # with same name does not match same ftrack id?
+ if "ftrackId" not in av_ent["data"]:
+ selected_av_entities.append(av_ent)
+ found_without_ftrack_id[str(av_ent["_id"])] = ftrack_id
+ break
if not selected_av_entities:
return {
- "success": False,
- "message": "Didn't found entities in avalon"
+ "success": True,
+ "message": (
+ "Didn't found entities in avalon."
+ " You can use Ftrack's Delete button for the selection."
+ )
}
# Remove cached action older than 2 minutes
old_action_ids = []
- for id, data in self.action_data_by_id.items():
+ for action_id, data in self.action_data_by_id.items():
created_at = data.get("created_at")
if not created_at:
- old_action_ids.append(id)
+ old_action_ids.append(action_id)
continue
cur_time = datetime.now()
existing_in_sec = (created_at - cur_time).total_seconds()
if existing_in_sec > 60 * 2:
- old_action_ids.append(id)
+ old_action_ids.append(action_id)
- for id in old_action_ids:
- self.action_data_by_id.pop(id, None)
+ for action_id in old_action_ids:
+ self.action_data_by_id.pop(action_id, None)
# Store data for action id
action_id = str(uuid.uuid1())
@@ -201,10 +208,7 @@ def interface(self, session, entities, event):
items.append(id_item)
asset_ids = [ent["_id"] for ent in selected_av_entities]
- subsets_for_selection = self.dbcon.find({
- "type": "subset",
- "parent": {"$in": asset_ids}
- })
+ subsets_for_selection = get_subsets(project_name, asset_ids=asset_ids)
asset_ending = ""
if len(selected_av_entities) > 1:
@@ -439,7 +443,11 @@ def launch(self, session, entities, event):
subsets_to_delete = to_delete.get("subsets") or []
# Convert asset ids to ObjectId obj
- assets_to_delete = [ObjectId(id) for id in assets_to_delete if id]
+ assets_to_delete = [
+ ObjectId(asset_id)
+ for asset_id in assets_to_delete
+ if asset_id
+ ]
subset_ids_by_parent = spec_data["subset_ids_by_parent"]
subset_ids_by_name = spec_data["subset_ids_by_name"]
@@ -450,13 +458,9 @@ def launch(self, session, entities, event):
if len(assets_to_delete) > 0:
map_av_ftrack_id = spec_data["without_ftrack_id"]
# Prepare data when deleting whole avalon asset
- avalon_assets = self.dbcon.find(
- {"type": "asset"},
- {
- "_id": 1,
- "data.visualParent": 1,
- "data.ftrackId": 1
- }
+ avalon_assets = get_assets(
+ project_name,
+ fields=["_id", "data.visualParent", "data.ftrackId"]
)
avalon_assets_by_parent = collections.defaultdict(list)
for asset in avalon_assets:
@@ -468,9 +472,8 @@ def launch(self, session, entities, event):
if not ftrack_id:
ftrack_id = asset["data"].get("ftrackId")
- if not ftrack_id:
- continue
- ftrack_ids_to_delete.append(ftrack_id)
+ if ftrack_id:
+ ftrack_ids_to_delete.append(ftrack_id)
children_queue = collections.deque()
for mongo_id in assets_to_delete:
@@ -548,8 +551,8 @@ def launch(self, session, entities, event):
ftrack_proc_txt, ", ".join(ftrack_ids_to_delete)
))
- entities_by_link_len = (
- self._filter_entities_to_delete(ftrack_ids_to_delete, session)
+ entities_by_link_len = self._prepare_entities_before_delete(
+ ftrack_ids_to_delete, session
)
for link_len in sorted(entities_by_link_len.keys(), reverse=True):
for entity in entities_by_link_len[link_len]:
@@ -569,12 +572,12 @@ def launch(self, session, entities, event):
exc_info=True
)
- if not_deleted_entities_id:
- joined_not_deleted = ", ".join([
+ if not_deleted_entities_id and asset_names_to_delete:
+ joined_not_deleted = ",".join([
"\"{}\"".format(ftrack_id)
for ftrack_id in not_deleted_entities_id
])
- joined_asset_names = ", ".join([
+ joined_asset_names = ",".join([
"\"{}\"".format(name)
for name in asset_names_to_delete
])
@@ -603,7 +606,7 @@ def launch(self, session, entities, event):
return self.report_handle(report_messages, project_name, event)
- def _filter_entities_to_delete(self, ftrack_ids_to_delete, session):
+ def _prepare_entities_before_delete(self, ftrack_ids_to_delete, session):
"""Filter children entities to avoid CircularDependencyError."""
joined_ids_to_delete = ", ".join(
["\"{}\"".format(id) for id in ftrack_ids_to_delete]
@@ -613,6 +616,40 @@ def _filter_entities_to_delete(self, ftrack_ids_to_delete, session):
joined_ids_to_delete
)
).all()
+ # Find all children entities and add them to list
+ # - Delete tasks first then their parents and continue
+ parent_ids_to_delete = [
+ entity["id"]
+ for entity in to_delete_entities
+ ]
+ while parent_ids_to_delete:
+ joined_parent_ids_to_delete = ",".join([
+ "\"{}\"".format(ftrack_id)
+ for ftrack_id in parent_ids_to_delete
+ ])
+ _to_delete = session.query((
+ "select id, link from TypedContext where parent_id in ({})"
+ ).format(joined_parent_ids_to_delete)).all()
+ parent_ids_to_delete = []
+ for entity in _to_delete:
+ parent_ids_to_delete.append(entity["id"])
+ to_delete_entities.append(entity)
+
+ # Unset 'task_id' from AssetVersion entities
+ # - when task is deleted the asset version is not marked for deletion
+ task_ids = set(
+ entity["id"]
+ for entity in to_delete_entities
+ if entity.entity_type.lower() == "task"
+ )
+ for chunk in create_chunks(task_ids):
+ asset_versions = session.query((
+ "select id, task_id from AssetVersion where task_id in ({})"
+ ).format(self.join_query_keys(chunk))).all()
+ for asset_version in asset_versions:
+ asset_version["task_id"] = None
+ session.commit()
+
entities_by_link_len = collections.defaultdict(list)
for entity in to_delete_entities:
entities_by_link_len[len(entity["link"])].append(entity)
diff --git a/openpype/modules/default_modules/ftrack/event_handlers_user/action_delete_old_versions.py b/openpype/modules/ftrack/event_handlers_user/action_delete_old_versions.py
similarity index 93%
rename from openpype/modules/default_modules/ftrack/event_handlers_user/action_delete_old_versions.py
rename to openpype/modules/ftrack/event_handlers_user/action_delete_old_versions.py
index c66d1819aca..c543dc8834f 100644
--- a/openpype/modules/default_modules/ftrack/event_handlers_user/action_delete_old_versions.py
+++ b/openpype/modules/ftrack/event_handlers_user/action_delete_old_versions.py
@@ -5,11 +5,19 @@
import clique
from pymongo import UpdateOne
+from openpype.client import (
+ get_assets,
+ get_subsets,
+ get_versions,
+ get_representations
+)
+from openpype.lib import (
+ StringTemplate,
+ TemplateUnsolved,
+ format_file_size,
+)
+from openpype.pipeline import AvalonMongoDB, Anatomy
from openpype_modules.ftrack.lib import BaseAction, statics_icon
-from avalon.api import AvalonMongoDB
-from openpype.api import Anatomy
-
-import avalon.pipeline
class DeleteOldVersions(BaseAction):
@@ -130,13 +138,6 @@ def interface(self, session, entities, event):
"title": self.inteface_title
}
- def sizeof_fmt(self, num, suffix='B'):
- for unit in ['', 'Ki', 'Mi', 'Gi', 'Ti', 'Pi', 'Ei', 'Zi']:
- if abs(num) < 1024.0:
- return "%3.1f%s%s" % (num, unit, suffix)
- num /= 1024.0
- return "%.1f%s%s" % (num, 'Yi', suffix)
-
def launch(self, session, entities, event):
values = event["data"].get("values")
if not values:
@@ -198,10 +199,9 @@ def launch(self, session, entities, event):
self.log.debug("Project is set to {}".format(project_name))
# Get Assets from avalon database
- assets = list(self.dbcon.find({
- "type": "asset",
- "name": {"$in": avalon_asset_names}
- }))
+ assets = list(
+ get_assets(project_name, asset_names=avalon_asset_names)
+ )
asset_id_to_name_map = {
asset["_id"]: asset["name"] for asset in assets
}
@@ -210,10 +210,9 @@ def launch(self, session, entities, event):
self.log.debug("Collected assets ({})".format(len(asset_ids)))
# Get Subsets
- subsets = list(self.dbcon.find({
- "type": "subset",
- "parent": {"$in": asset_ids}
- }))
+ subsets = list(
+ get_subsets(project_name, asset_ids=asset_ids)
+ )
subsets_by_id = {}
subset_ids = []
for subset in subsets:
@@ -230,10 +229,9 @@ def launch(self, session, entities, event):
self.log.debug("Collected subsets ({})".format(len(subset_ids)))
# Get Versions
- versions = list(self.dbcon.find({
- "type": "version",
- "parent": {"$in": subset_ids}
- }))
+ versions = list(
+ get_versions(project_name, subset_ids=subset_ids)
+ )
versions_by_parent = collections.defaultdict(list)
for ent in versions:
@@ -295,10 +293,9 @@ def sort_func(ent):
"message": msg
}
- repres = list(self.dbcon.find({
- "type": "representation",
- "parent": {"$in": version_ids}
- }))
+ repres = list(
+ get_representations(project_name, version_ids=version_ids)
+ )
self.log.debug(
"Collected representations to remove ({})".format(len(repres))
@@ -359,7 +356,7 @@ def sort_func(ent):
dir_paths, file_paths_by_dir, delete=False
)
- msg = "Total size of files: " + self.sizeof_fmt(size)
+ msg = "Total size of files: {}".format(format_file_size(size))
self.log.warning(msg)
@@ -430,7 +427,7 @@ def sort_func(ent):
"message": msg
}
- msg = "Total size of files deleted: " + self.sizeof_fmt(size)
+ msg = "Total size of files deleted: {}".format(format_file_size(size))
self.log.warning(msg)
@@ -492,7 +489,8 @@ def delete_only_repre_files(self, dir_paths, file_paths, delete=True):
os.remove(file_path)
self.log.debug("Removed file: {}".format(file_path))
- remainders.remove(file_path_base)
+ if file_path_base in remainders:
+ remainders.remove(file_path_base)
continue
seq_path_base = os.path.split(seq_path)[1]
@@ -563,18 +561,16 @@ def path_from_represenation(self, representation, anatomy):
try:
context = representation["context"]
context["root"] = anatomy.roots
- path = avalon.pipeline.format_template_with_optional_keys(
- context, template
- )
+ path = StringTemplate.format_strict_template(template, context)
if "frame" in context:
context["frame"] = self.sequence_splitter
sequence_path = os.path.normpath(
- avalon.pipeline.format_template_with_optional_keys(
- context, template
+ StringTemplate.format_strict_template(
+ template, context
)
)
- except KeyError:
+ except (KeyError, TemplateUnsolved):
# Template references unavailable data
return (None, None)
diff --git a/openpype/modules/ftrack/event_handlers_user/action_delivery.py b/openpype/modules/ftrack/event_handlers_user/action_delivery.py
new file mode 100644
index 00000000000..a400c8f5f03
--- /dev/null
+++ b/openpype/modules/ftrack/event_handlers_user/action_delivery.py
@@ -0,0 +1,645 @@
+import os
+import copy
+import json
+import collections
+
+from openpype.client import (
+ get_project,
+ get_assets,
+ get_subsets,
+ get_versions,
+ get_representations
+)
+from openpype_modules.ftrack.lib import BaseAction, statics_icon
+from openpype_modules.ftrack.lib.avalon_sync import CUST_ATTR_ID_KEY
+from openpype_modules.ftrack.lib.custom_attributes import (
+ query_custom_attributes
+)
+from openpype.lib.dateutils import get_datetime_data
+from openpype.pipeline import Anatomy
+from openpype.pipeline.load import get_representation_path_with_anatomy
+from openpype.pipeline.delivery import (
+ get_format_dict,
+ check_destination_path,
+ deliver_single_file,
+ deliver_sequence,
+)
+
+
+class Delivery(BaseAction):
+ identifier = "delivery.action"
+ label = "Delivery"
+ description = "Deliver data to client"
+ role_list = ["Pypeclub", "Administrator", "Project manager"]
+ icon = statics_icon("ftrack", "action_icons", "Delivery.svg")
+ settings_key = "delivery_action"
+
+ def discover(self, session, entities, event):
+ is_valid = False
+ for entity in entities:
+ if entity.entity_type.lower() in ("assetversion", "reviewsession"):
+ is_valid = True
+ break
+
+ if is_valid:
+ is_valid = self.valid_roles(session, entities, event)
+ return is_valid
+
+ def interface(self, session, entities, event):
+ if event["data"].get("values", {}):
+ return
+
+ title = "Delivery data to Client"
+
+ items = []
+ item_splitter = {"type": "label", "value": "---"}
+
+ project_entity = self.get_project_from_entity(entities[0])
+ project_name = project_entity["full_name"]
+ project_doc = get_project(project_name, fields=["name"])
+ if not project_doc:
+ return {
+ "success": False,
+ "message": (
+ "Didn't found project \"{}\" in avalon."
+ ).format(project_name)
+ }
+
+ repre_names = self._get_repre_names(project_name, session, entities)
+
+ items.append({
+ "type": "hidden",
+ "name": "__project_name__",
+ "value": project_name
+ })
+
+ # Prepare anatomy data
+ anatomy = Anatomy(project_name)
+ new_anatomies = []
+ first = None
+ for key, template in (anatomy.templates.get("delivery") or {}).items():
+ # Use only keys with `{root}` or `{root[*]}` in value
+ if isinstance(template, str) and "{root" in template:
+ new_anatomies.append({
+ "label": key,
+ "value": key
+ })
+ if first is None:
+ first = key
+
+ skipped = False
+ # Add message if there are any common components
+ if not repre_names or not new_anatomies:
+ skipped = True
+ items.append({
+ "type": "label",
+ "value": "Something went wrong: "
+ })
+
+ items.append({
+ "type": "hidden",
+ "name": "__skipped__",
+ "value": skipped
+ })
+
+ if not repre_names:
+ if len(entities) == 1:
+ items.append({
+ "type": "label",
+ "value": (
+ "- Selected entity doesn't have components to deliver."
+ )
+ })
+ else:
+ items.append({
+ "type": "label",
+ "value": (
+ "- Selected entities don't have common components."
+ )
+ })
+
+ # Add message if delivery anatomies are not set
+ if not new_anatomies:
+ items.append({
+ "type": "label",
+ "value": (
+ "- `\"delivery\"` anatomy key is not set in config."
+ )
+ })
+
+ # Skip if there are any data shortcomings
+ if skipped:
+ return {
+ "items": items,
+ "title": title
+ }
+
+ items.append({
+ "value": "Choose Components to deliver ",
+ "type": "label"
+ })
+
+ for repre_name in repre_names:
+ items.append({
+ "type": "boolean",
+ "value": False,
+ "label": repre_name,
+ "name": repre_name
+ })
+
+ items.append(item_splitter)
+
+ items.append({
+ "value": "Location for delivery ",
+ "type": "label"
+ })
+
+ items.append({
+ "type": "label",
+ "value": (
+ "NOTE: It is possible to replace `root` key in anatomy. "
+ )
+ })
+
+ items.append({
+ "type": "text",
+ "name": "__location_path__",
+ "empty_text": "Type location path here...(Optional)"
+ })
+
+ items.append(item_splitter)
+
+ items.append({
+ "value": "Anatomy of delivery files ",
+ "type": "label"
+ })
+
+ items.append({
+ "type": "label",
+ "value": (
+ "NOTE: These can be set in Anatomy.yaml"
+ " within `delivery` key.
"
+ )
+ })
+
+ items.append({
+ "type": "enumerator",
+ "name": "__new_anatomies__",
+ "data": new_anatomies,
+ "value": first
+ })
+
+ return {
+ "items": items,
+ "title": title
+ }
+
+ def _get_repre_names(self, project_name, session, entities):
+ version_ids = self._get_interest_version_ids(
+ project_name, session, entities
+ )
+ if not version_ids:
+ return []
+ repre_docs = get_representations(
+ project_name,
+ version_ids=version_ids,
+ fields=["name"]
+ )
+ repre_names = {repre_doc["name"] for repre_doc in repre_docs}
+ return list(sorted(repre_names))
+
+ def _get_interest_version_ids(self, project_name, session, entities):
+ # Extract AssetVersion entities
+ asset_versions = self._extract_asset_versions(session, entities)
+ # Prepare Asset ids
+ asset_ids = {
+ asset_version["asset_id"]
+ for asset_version in asset_versions
+ }
+ # Query Asset entities
+ assets = session.query((
+ "select id, name, context_id from Asset where id in ({})"
+ ).format(self.join_query_keys(asset_ids))).all()
+ assets_by_id = {
+ asset["id"]: asset
+ for asset in assets
+ }
+ parent_ids = set()
+ subset_names = set()
+ version_nums = set()
+ for asset_version in asset_versions:
+ asset_id = asset_version["asset_id"]
+ asset = assets_by_id[asset_id]
+
+ parent_ids.add(asset["context_id"])
+ subset_names.add(asset["name"])
+ version_nums.add(asset_version["version"])
+
+ asset_docs_by_ftrack_id = self._get_asset_docs(
+ project_name, session, parent_ids
+ )
+ subset_docs = self._get_subset_docs(
+ project_name,
+ asset_docs_by_ftrack_id,
+ subset_names,
+ asset_versions,
+ assets_by_id
+ )
+ version_docs = self._get_version_docs(
+ project_name,
+ asset_docs_by_ftrack_id,
+ subset_docs,
+ version_nums,
+ asset_versions,
+ assets_by_id
+ )
+
+ return [version_doc["_id"] for version_doc in version_docs]
+
+ def _extract_asset_versions(self, session, entities):
+ asset_version_ids = set()
+ review_session_ids = set()
+ for entity in entities:
+ entity_type_low = entity.entity_type.lower()
+ if entity_type_low == "assetversion":
+ asset_version_ids.add(entity["id"])
+ elif entity_type_low == "reviewsession":
+ review_session_ids.add(entity["id"])
+
+ for version_id in self._get_asset_version_ids_from_review_sessions(
+ session, review_session_ids
+ ):
+ asset_version_ids.add(version_id)
+
+ asset_versions = session.query((
+ "select id, version, asset_id from AssetVersion where id in ({})"
+ ).format(self.join_query_keys(asset_version_ids))).all()
+
+ return asset_versions
+
+ def _get_asset_version_ids_from_review_sessions(
+ self, session, review_session_ids
+ ):
+ if not review_session_ids:
+ return set()
+ review_session_objects = session.query((
+ "select version_id from ReviewSessionObject"
+ " where review_session_id in ({})"
+ ).format(self.join_query_keys(review_session_ids))).all()
+
+ return {
+ review_session_object["version_id"]
+ for review_session_object in review_session_objects
+ }
+
+ def _get_version_docs(
+ self,
+ project_name,
+ asset_docs_by_ftrack_id,
+ subset_docs,
+ version_nums,
+ asset_versions,
+ assets_by_id
+ ):
+ subset_docs_by_id = {
+ subset_doc["_id"]: subset_doc
+ for subset_doc in subset_docs
+ }
+ version_docs = list(get_versions(
+ project_name,
+ subset_ids=subset_docs_by_id.keys(),
+ versions=version_nums
+ ))
+ version_docs_by_parent_id = collections.defaultdict(dict)
+ for version_doc in version_docs:
+ subset_doc = subset_docs_by_id[version_doc["parent"]]
+
+ asset_id = subset_doc["parent"]
+ subset_name = subset_doc["name"]
+ version = version_doc["name"]
+ if version_docs_by_parent_id[asset_id].get(subset_name) is None:
+ version_docs_by_parent_id[asset_id][subset_name] = {}
+
+ version_docs_by_parent_id[asset_id][subset_name][version] = (
+ version_doc
+ )
+
+ filtered_versions = []
+ for asset_version in asset_versions:
+ asset_id = asset_version["asset_id"]
+ asset = assets_by_id[asset_id]
+ parent_id = asset["context_id"]
+ asset_doc = asset_docs_by_ftrack_id.get(parent_id)
+ if not asset_doc:
+ continue
+
+ subsets_by_name = version_docs_by_parent_id.get(asset_doc["_id"])
+ if not subsets_by_name:
+ continue
+
+ subset_name = asset["name"]
+ version_docs_by_version = subsets_by_name.get(subset_name)
+ if not version_docs_by_version:
+ continue
+
+ version = asset_version["version"]
+ version_doc = version_docs_by_version.get(version)
+ if version_doc:
+ filtered_versions.append(version_doc)
+ return filtered_versions
+
+ def _get_subset_docs(
+ self,
+ project_name,
+ asset_docs_by_ftrack_id,
+ subset_names,
+ asset_versions,
+ assets_by_id
+ ):
+ asset_doc_ids = [
+ asset_doc["_id"]
+ for asset_doc in asset_docs_by_ftrack_id.values()
+ ]
+ subset_docs = list(get_subsets(
+ project_name,
+ asset_ids=asset_doc_ids,
+ subset_names=subset_names
+ ))
+ subset_docs_by_parent_id = collections.defaultdict(dict)
+ for subset_doc in subset_docs:
+ asset_id = subset_doc["parent"]
+ subset_name = subset_doc["name"]
+ subset_docs_by_parent_id[asset_id][subset_name] = subset_doc
+
+ filtered_subsets = []
+ for asset_version in asset_versions:
+ asset_id = asset_version["asset_id"]
+ asset = assets_by_id[asset_id]
+
+ parent_id = asset["context_id"]
+ asset_doc = asset_docs_by_ftrack_id.get(parent_id)
+ if not asset_doc:
+ continue
+
+ subsets_by_name = subset_docs_by_parent_id.get(asset_doc["_id"])
+ if not subsets_by_name:
+ continue
+
+ subset_name = asset["name"]
+ subset_doc = subsets_by_name.get(subset_name)
+ if subset_doc:
+ filtered_subsets.append(subset_doc)
+ return filtered_subsets
+
+ def _get_asset_docs(self, project_name, session, parent_ids):
+ asset_docs = list(get_assets(
+ project_name, fields=["_id", "name", "data.ftrackId"]
+ ))
+
+ asset_docs_by_id = {}
+ asset_docs_by_name = {}
+ asset_docs_by_ftrack_id = {}
+ for asset_doc in asset_docs:
+ asset_id = str(asset_doc["_id"])
+ asset_name = asset_doc["name"]
+ ftrack_id = asset_doc["data"].get("ftrackId")
+
+ asset_docs_by_id[asset_id] = asset_doc
+ asset_docs_by_name[asset_name] = asset_doc
+ if ftrack_id:
+ asset_docs_by_ftrack_id[ftrack_id] = asset_doc
+
+ attr_def = session.query((
+ "select id from CustomAttributeConfiguration where key is \"{}\""
+ ).format(CUST_ATTR_ID_KEY)).first()
+ if attr_def is None:
+ return asset_docs_by_ftrack_id
+
+ avalon_mongo_id_values = query_custom_attributes(
+ session, [attr_def["id"]], parent_ids, True
+ )
+ missing_ids = set(parent_ids)
+ for item in avalon_mongo_id_values:
+ if not item["value"]:
+ continue
+ asset_id = item["value"]
+ entity_id = item["entity_id"]
+ asset_doc = asset_docs_by_id.get(asset_id)
+ if asset_doc:
+ asset_docs_by_ftrack_id[entity_id] = asset_doc
+ missing_ids.remove(entity_id)
+
+ entity_ids_by_name = {}
+ if missing_ids:
+ not_found_entities = session.query((
+ "select id, name from TypedContext where id in ({})"
+ ).format(self.join_query_keys(missing_ids))).all()
+ entity_ids_by_name = {
+ entity["name"]: entity["id"]
+ for entity in not_found_entities
+ }
+
+ for asset_name, entity_id in entity_ids_by_name.items():
+ asset_doc = asset_docs_by_name.get(asset_name)
+ if asset_doc:
+ asset_docs_by_ftrack_id[entity_id] = asset_doc
+
+ return asset_docs_by_ftrack_id
+
+ def launch(self, session, entities, event):
+ if "values" not in event["data"]:
+ return {
+ "success": True,
+ "message": "Nothing to do"
+ }
+
+ values = event["data"]["values"]
+ skipped = values.pop("__skipped__")
+ if skipped:
+ return {
+ "success": False,
+ "message": "Action skipped"
+ }
+
+ user_id = event["source"]["user"]["id"]
+ user_entity = session.query(
+ "User where id is {}".format(user_id)
+ ).one()
+
+ job = session.create("Job", {
+ "user": user_entity,
+ "status": "running",
+ "data": json.dumps({
+ "description": "Delivery processing."
+ })
+ })
+ session.commit()
+
+ try:
+ report = self.real_launch(session, entities, event)
+
+ except Exception as exc:
+ report = {
+ "success": False,
+ "title": "Delivery failed",
+ "items": [{
+ "type": "label",
+ "value": (
+ "Error during delivery action process:{}
'.format("{}
'.format("Set basic Attributes: "
+ })
+
+ items.extend(ca_items)
+
+ # Set value of auto synchronization
+ auto_sync_value = project_entity["custom_attributes"].get(
+ CUST_ATTR_AUTO_SYNC, False
+ )
+ auto_sync_item = {
+ "name": CUST_ATTR_AUTO_SYNC,
+ "type": "boolean",
+ "value": auto_sync_value,
+ "label": "AutoSync to Avalon"
+ }
+ # Add autosync attribute
+ items.append(auto_sync_item)
+
+ # This item will be last before enumerators
+ # Ask if want to trigger Action Create Folder Structure
+ create_project_structure_checked = (
+ project_settings
+ ["project_settings"]
+ ["ftrack"]
+ ["user_handlers"]
+ ["prepare_project"]
+ ["create_project_structure_checked"]
+ ).value
+ items.append({
+ "type": "label",
+ "value": "Want to create basic Folder Structure? "
+ })
+ items.append({
+ "name": self.create_project_structure_key,
+ "type": "boolean",
+ "value": create_project_structure_checked,
+ "label": "Check if Yes"
+ })
+
+ # Add enumerator items at the end
+ for item in multiselect_enumerators:
+ items.append(item)
+
+ return {
+ "items": items,
+ "title": title
+ }
+
+ def prepare_root_items(self, project_anatom_settings):
+ self.log.debug("Root items preparation begins.")
+
+ root_items = []
+ root_items.append({
+ "type": "label",
+ "value": "Check your Project root settings "
+ })
+ root_items.append({
+ "type": "label",
+ "value": (
+ "NOTE: Roots are crutial for path filling"
+ " (and creating folder structure).
"
+ )
+ })
+ root_items.append({
+ "type": "label",
+ "value": (
+ "WARNING: Do not change roots on running project,"
+ " that will cause workflow issues .
"
+ )
+ })
+
+ empty_text = "Enter root path here..."
+
+ roots_entity = project_anatom_settings["roots"]
+ for root_name, root_entity in roots_entity.items():
+ root_items.append(self.item_splitter)
+ root_items.append({
+ "type": "label",
+ "value": "Root: \"{}\"".format(root_name)
+ })
+ for platform_name, value_entity in root_entity.items():
+ root_items.append({
+ "label": platform_name,
+ "name": "__root__{}__{}".format(root_name, platform_name),
+ "type": "text",
+ "value": value_entity.value,
+ "empty_text": empty_text
+ })
+
+ root_items.append({
+ "type": "hidden",
+ "name": "__rootnames__",
+ "value": json.dumps(list(roots_entity.keys()))
+ })
+
+ self.log.debug("Root items preparation ended.")
+ return root_items
+
+ def _attributes_to_set(self, project_anatom_settings):
+ attributes_to_set = {}
+
+ attribute_values_by_key = {}
+ for key, entity in project_anatom_settings["attributes"].items():
+ attribute_values_by_key[key] = entity.value
+
+ cust_attrs, hier_cust_attrs = get_openpype_attr(self.session, True)
+
+ for attr in hier_cust_attrs:
+ key = attr["key"]
+ if key.startswith("avalon_"):
+ continue
+ attributes_to_set[key] = {
+ "label": attr["label"],
+ "object": attr,
+ "default": attribute_values_by_key.get(key)
+ }
+
+ for attr in cust_attrs:
+ if attr["entity_type"].lower() != "show":
+ continue
+ key = attr["key"]
+ if key.startswith("avalon_"):
+ continue
+ attributes_to_set[key] = {
+ "label": attr["label"],
+ "object": attr,
+ "default": attribute_values_by_key.get(key)
+ }
+
+ # Sort by label
+ attributes_to_set = dict(sorted(
+ attributes_to_set.items(),
+ key=lambda x: x[1]["label"]
+ ))
+ return attributes_to_set
+
+ def prepare_custom_attribute_items(self, project_anatom_settings):
+ items = []
+ multiselect_enumerators = []
+ attributes_to_set = self._attributes_to_set(project_anatom_settings)
+
+ self.log.debug("Preparing interface for keys: \"{}\"".format(
+ str([key for key in attributes_to_set])
+ ))
+
+ attribute_keys = set(attributes_to_set.keys())
+ keys_order = []
+ for key in self._keys_order:
+ if key in attribute_keys:
+ keys_order.append(key)
+
+ attribute_keys = attribute_keys - set(keys_order)
+ for key in sorted(attribute_keys):
+ keys_order.append(key)
+
+ for key in keys_order:
+ in_data = attributes_to_set[key]
+ attr = in_data["object"]
+
+ # initial item definition
+ item = {
+ "name": key,
+ "label": in_data["label"]
+ }
+
+ # cust attr type - may have different visualization
+ type_name = attr["type"]["name"].lower()
+ easy_types = ["text", "boolean", "date", "number"]
+
+ easy_type = False
+ if type_name in easy_types:
+ easy_type = True
+
+ elif type_name == "enumerator":
+
+ attr_config = json.loads(attr["config"])
+ attr_config_data = json.loads(attr_config["data"])
+
+ if attr_config["multiSelect"] is True:
+ multiselect_enumerators.append(self.item_splitter)
+ multiselect_enumerators.append({
+ "type": "label",
+ "value": "{} ".format(in_data["label"])
+ })
+
+ default = in_data["default"]
+ names = []
+ for option in sorted(
+ attr_config_data, key=lambda x: x["menu"]
+ ):
+ name = option["value"]
+ new_name = "__{}__{}".format(key, name)
+ names.append(new_name)
+ item = {
+ "name": new_name,
+ "type": "boolean",
+ "label": "- {}".format(option["menu"])
+ }
+ if default:
+ if isinstance(default, (list, tuple)):
+ if name in default:
+ item["value"] = True
+ else:
+ if name == default:
+ item["value"] = True
+
+ multiselect_enumerators.append(item)
+
+ multiselect_enumerators.append({
+ "type": "hidden",
+ "name": "__hidden__{}".format(key),
+ "value": json.dumps(names)
+ })
+ else:
+ easy_type = True
+ item["data"] = attr_config_data
+
+ else:
+ self.log.warning((
+ "Custom attribute \"{}\" has type \"{}\"."
+ " I don't know how to handle"
+ ).format(key, type_name))
+ items.append({
+ "type": "label",
+ "value": (
+ "!!! Can't handle Custom attritubte type \"{}\""
+ " (key: \"{}\")"
+ ).format(type_name, key)
+ })
+
+ if easy_type:
+ item["type"] = type_name
+
+ # default value in interface
+ default = in_data["default"]
+ if default is not None:
+ item["value"] = default
+
+ items.append(item)
+
+ return items, multiselect_enumerators
+
+ def launch(self, session, entities, event):
+ in_data = event["data"].get("values")
+ if not in_data:
+ return
+
+ create_project_structure_checked = in_data.pop(
+ self.create_project_structure_key
+ )
+
+ root_values = {}
+ root_key = "__root__"
+ for key in tuple(in_data.keys()):
+ if key.startswith(root_key):
+ _key = key[len(root_key):]
+ root_values[_key] = in_data.pop(key)
+
+ root_names = in_data.pop("__rootnames__", None)
+ root_data = {}
+ for root_name in json.loads(root_names):
+ root_data[root_name] = {}
+ for key, value in tuple(root_values.items()):
+ prefix = "{}__".format(root_name)
+ if not key.startswith(prefix):
+ continue
+
+ _key = key[len(prefix):]
+ root_data[root_name][_key] = value
+
+ # Find hidden items for multiselect enumerators
+ keys_to_process = []
+ for key in in_data:
+ if key.startswith("__hidden__"):
+ keys_to_process.append(key)
+
+ self.log.debug("Preparing data for Multiselect Enumerators")
+ enumerators = {}
+ for key in keys_to_process:
+ new_key = key.replace("__hidden__", "")
+ enumerator_items = in_data.pop(key)
+ enumerators[new_key] = json.loads(enumerator_items)
+
+ # find values set for multiselect enumerator
+ for key, enumerator_items in enumerators.items():
+ in_data[key] = []
+
+ name = "__{}__".format(key)
+
+ for item in enumerator_items:
+ value = in_data.pop(item)
+ if value is True:
+ new_key = item.replace(name, "")
+ in_data[key].append(new_key)
+
+ self.log.debug("Setting Custom Attribute values")
+
+ project_entity = entities[0]
+ project_name = project_entity["full_name"]
+
+ # Try to find project document
+ project_doc = get_project(project_name)
+
+ # Create project if is not available
+ # - creation is required to be able set project anatomy and attributes
+ if not project_doc:
+ project_code = project_entity["name"]
+ self.log.info("Creating project \"{} [{}]\"".format(
+ project_name, project_code
+ ))
+ create_project(project_name, project_code)
+ self.trigger_event(
+ "openpype.project.created",
+ {"project_name": project_name}
+ )
+
+ project_settings = ProjectSettings(project_name)
+ project_anatomy_settings = project_settings["project_anatomy"]
+ project_anatomy_settings["roots"] = root_data
+
+ custom_attribute_values = {}
+ attributes_entity = project_anatomy_settings["attributes"]
+ for key, value in in_data.items():
+ if key not in attributes_entity:
+ custom_attribute_values[key] = value
+ else:
+ attributes_entity[key] = value
+
+ try:
+ project_settings.save()
+ except SaveWarningExc as exc:
+ self.log.info("Few warnings happened during settings save:")
+ for warning in exc.warnings:
+ self.log.info(str(warning))
+
+ # Change custom attributes on project
+ if custom_attribute_values:
+ for key, value in custom_attribute_values.items():
+ project_entity["custom_attributes"][key] = value
+ self.log.debug("- Key \"{}\" set to \"{}\"".format(key, value))
+ session.commit()
+
+ # Trigger create project structure action
+ if create_project_structure_checked:
+ trigger_identifier = "{}.{}".format(
+ self.create_project_structure_identifier,
+ self.process_identifier()
+ )
+ self.trigger_action(trigger_identifier, event)
+
+ event_data = copy.deepcopy(in_data)
+ event_data["project_name"] = project_name
+ self.trigger_event("openpype.project.prepared", event_data)
+ return True
+
+
+def register(session):
+ '''Register plugin. Called when used as an plugin.'''
+ PrepareProjectLocal(session).register()
diff --git a/openpype/modules/default_modules/ftrack/event_handlers_user/action_rv.py b/openpype/modules/ftrack/event_handlers_user/action_rv.py
similarity index 84%
rename from openpype/modules/default_modules/ftrack/event_handlers_user/action_rv.py
rename to openpype/modules/ftrack/event_handlers_user/action_rv.py
index 71d790f7e76..d05f0c47f6d 100644
--- a/openpype/modules/default_modules/ftrack/event_handlers_user/action_rv.py
+++ b/openpype/modules/ftrack/event_handlers_user/action_rv.py
@@ -3,9 +3,20 @@
import traceback
import json
-from openpype_modules.ftrack.lib import BaseAction, statics_icon
import ftrack_api
-from avalon import io, api
+
+from openpype.client import (
+ get_asset_by_name,
+ get_subset_by_name,
+ get_version_by_name,
+ get_representation_by_name
+)
+from openpype.pipeline import (
+ get_representation_path,
+ AvalonMongoDB,
+ Anatomy,
+)
+from openpype_modules.ftrack.lib import BaseAction, statics_icon
class RVAction(BaseAction):
@@ -251,9 +262,10 @@ def get_file_paths(self, session, event):
"Component", list(event["data"]["values"].values())[0]
)["version"]["asset"]["parent"]["link"][0]
project = session.get(link["type"], link["id"])
- os.environ["AVALON_PROJECT"] = project["name"]
- api.Session["AVALON_PROJECT"] = project["name"]
- io.install()
+ project_name = project["full_name"]
+ dbcon = AvalonMongoDB()
+ dbcon.Session["AVALON_PROJECT"] = project_name
+ anatomy = Anatomy(project_name)
location = ftrack_api.Session().pick_location()
@@ -277,37 +289,38 @@ def get_file_paths(self, session, event):
if online_source:
continue
- asset = io.find_one({"type": "asset", "name": parent_name})
- subset = io.find_one(
- {
- "type": "subset",
- "name": component["version"]["asset"]["name"],
- "parent": asset["_id"]
- }
+ subset_name = component["version"]["asset"]["name"]
+ version_name = component["version"]["version"]
+ representation_name = component["file_type"][1:]
+
+ asset_doc = get_asset_by_name(
+ project_name, parent_name, fields=["_id"]
)
- version = io.find_one(
- {
- "type": "version",
- "name": component["version"]["version"],
- "parent": subset["_id"]
- }
+ subset_doc = get_subset_by_name(
+ project_name,
+ subset_name=subset_name,
+ asset_id=asset_doc["_id"]
)
- representation = io.find_one(
- {
- "type": "representation",
- "parent": version["_id"],
- "name": component["file_type"][1:]
- }
+ version_doc = get_version_by_name(
+ project_name,
+ version=version_name,
+ subset_id=subset_doc["_id"]
)
- if representation is None:
- representation = io.find_one(
- {
- "type": "representation",
- "parent": version["_id"],
- "name": "preview"
- }
+ repre_doc = get_representation_by_name(
+ project_name,
+ version_id=version_doc["_id"],
+ representation_name=representation_name
+ )
+ if not repre_doc:
+ repre_doc = get_representation_by_name(
+ project_name,
+ version_id=version_doc["_id"],
+ representation_name="preview"
)
- paths.append(api.get_representation_path(representation))
+
+ paths.append(get_representation_path(
+ repre_doc, root=anatomy.roots, dbcon=dbcon
+ ))
return paths
diff --git a/openpype/modules/default_modules/ftrack/event_handlers_user/action_seed.py b/openpype/modules/ftrack/event_handlers_user/action_seed.py
similarity index 100%
rename from openpype/modules/default_modules/ftrack/event_handlers_user/action_seed.py
rename to openpype/modules/ftrack/event_handlers_user/action_seed.py
diff --git a/openpype/modules/default_modules/ftrack/event_handlers_user/action_store_thumbnails_to_avalon.py b/openpype/modules/ftrack/event_handlers_user/action_store_thumbnails_to_avalon.py
similarity index 92%
rename from openpype/modules/default_modules/ftrack/event_handlers_user/action_store_thumbnails_to_avalon.py
rename to openpype/modules/ftrack/event_handlers_user/action_store_thumbnails_to_avalon.py
index 4820925844b..8748f426bdc 100644
--- a/openpype/modules/default_modules/ftrack/event_handlers_user/action_store_thumbnails_to_avalon.py
+++ b/openpype/modules/ftrack/event_handlers_user/action_store_thumbnails_to_avalon.py
@@ -4,9 +4,17 @@
import requests
from bson.objectid import ObjectId
+
+from openpype.client import (
+ get_project,
+ get_asset_by_id,
+ get_assets,
+ get_subset_by_name,
+ get_version_by_name,
+ get_representations
+)
from openpype_modules.ftrack.lib import BaseAction, statics_icon
-from openpype.api import Anatomy
-from avalon.api import AvalonMongoDB
+from openpype.pipeline import AvalonMongoDB, Anatomy
from openpype_modules.ftrack.lib.avalon_sync import CUST_ATTR_ID_KEY
@@ -384,7 +392,7 @@ def get_avalon_entities_for_assetversion(self, asset_version, db_con):
db_con.Session["AVALON_PROJECT"] = project_name
- avalon_project = db_con.find_one({"type": "project"})
+ avalon_project = get_project(project_name)
output["project"] = avalon_project
if not avalon_project:
@@ -398,19 +406,17 @@ def get_avalon_entities_for_assetversion(self, asset_version, db_con):
asset_mongo_id = parent["custom_attributes"].get(CUST_ATTR_ID_KEY)
if asset_mongo_id:
try:
- asset_mongo_id = ObjectId(asset_mongo_id)
- asset_ent = db_con.find_one({
- "type": "asset",
- "_id": asset_mongo_id
- })
+ asset_ent = get_asset_by_id(project_name, asset_mongo_id)
except Exception:
pass
if not asset_ent:
- asset_ent = db_con.find_one({
- "type": "asset",
- "data.ftrackId": parent["id"]
- })
+ asset_docs = get_assets(project_name, asset_names=[parent["name"]])
+ for asset_doc in asset_docs:
+ ftrack_id = asset_doc.get("data", {}).get("ftrackId")
+ if ftrack_id == parent["id"]:
+ asset_ent = asset_doc
+ break
output["asset"] = asset_ent
@@ -421,13 +427,11 @@ def get_avalon_entities_for_assetversion(self, asset_version, db_con):
)
return output
- asset_mongo_id = asset_ent["_id"]
-
- subset_ent = db_con.find_one({
- "type": "subset",
- "parent": asset_mongo_id,
- "name": subset_name
- })
+ subset_ent = get_subset_by_name(
+ project_name,
+ subset_name=subset_name,
+ asset_id=asset_ent["_id"]
+ )
output["subset"] = subset_ent
@@ -438,11 +442,11 @@ def get_avalon_entities_for_assetversion(self, asset_version, db_con):
).format(subset_name, ent_path)
return output
- version_ent = db_con.find_one({
- "type": "version",
- "name": version,
- "parent": subset_ent["_id"]
- })
+ version_ent = get_version_by_name(
+ project_name,
+ version,
+ subset_ent["_id"]
+ )
output["version"] = version_ent
@@ -453,10 +457,10 @@ def get_avalon_entities_for_assetversion(self, asset_version, db_con):
).format(version, subset_name, ent_path)
return output
- repre_ents = list(db_con.find({
- "type": "representation",
- "parent": version_ent["_id"]
- }))
+ repre_ents = list(get_representations(
+ project_name,
+ version_ids=[version_ent["_id"]]
+ ))
output["representations"] = repre_ents
return output
diff --git a/openpype/modules/ftrack/event_handlers_user/action_sync_to_avalon.py b/openpype/modules/ftrack/event_handlers_user/action_sync_to_avalon.py
new file mode 100644
index 00000000000..e52a061471f
--- /dev/null
+++ b/openpype/modules/ftrack/event_handlers_user/action_sync_to_avalon.py
@@ -0,0 +1,225 @@
+import time
+import sys
+import json
+
+import ftrack_api
+
+from openpype_modules.ftrack.lib import BaseAction, statics_icon
+from openpype_modules.ftrack.lib.avalon_sync import SyncEntitiesFactory
+
+
+class SyncToAvalonLocal(BaseAction):
+ """
+ Synchronizing data action - from Ftrack to Avalon DB
+
+ Stores all information about entity.
+ - Name(string) - Most important information = identifier of entity
+ - Parent(ObjectId) - Avalon Project Id, if entity is not project itself
+ - Data(dictionary):
+ - VisualParent(ObjectId) - Avalon Id of parent asset
+ - Parents(array of string) - All parent names except project
+ - Tasks(array of string) - Tasks on asset
+ - FtrackId(string)
+ - entityType(string) - entity's type on Ftrack
+ * All Custom attributes in group 'Avalon'
+ - custom attributes that start with 'avalon_' are skipped
+
+ * These information are stored for entities in whole project.
+
+ Avalon ID of asset is stored to Ftrack
+ - Custom attribute 'avalon_mongo_id'.
+ - action IS NOT creating this Custom attribute if doesn't exist
+ - run 'Create Custom Attributes' action
+ - or do it manually (Not recommended)
+ """
+
+ identifier = "sync.to.avalon.local"
+ label = "OpenPype Admin"
+ variant = "- Sync To Avalon (Local)"
+ priority = 200
+ icon = statics_icon("ftrack", "action_icons", "OpenPypeAdmin.svg")
+
+ settings_key = "sync_to_avalon_local"
+
+ def __init__(self, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ self.entities_factory = SyncEntitiesFactory(self.log, self.session)
+
+ def discover(self, session, entities, event):
+ """ Validate selection. """
+ is_valid = False
+ for ent in event["data"]["selection"]:
+ # Ignore entities that are not tasks or projects
+ if ent["entityType"].lower() in ["show", "task"]:
+ is_valid = True
+ break
+
+ if is_valid:
+ is_valid = self.valid_roles(session, entities, event)
+ return is_valid
+
+ def launch(self, session, in_entities, event):
+ self.log.debug("{}: Creating job".format(self.label))
+
+ user_entity = session.query(
+ "User where id is {}".format(event["source"]["user"]["id"])
+ ).one()
+ job_entity = session.create("Job", {
+ "user": user_entity,
+ "status": "running",
+ "data": json.dumps({
+ "description": "Sync to avalon is running..."
+ })
+ })
+ session.commit()
+
+ project_entity = self.get_project_from_entity(in_entities[0])
+ project_name = project_entity["full_name"]
+
+ try:
+ result = self.synchronization(event, project_name)
+
+ except Exception:
+ self.log.error(
+ "Synchronization failed due to code error", exc_info=True
+ )
+
+ description = "Sync to avalon Crashed (Download traceback)"
+ self.add_traceback_to_job(
+ job_entity, session, sys.exc_info(), description
+ )
+
+ msg = "An error has happened during synchronization"
+ title = "Synchronization report ({}):".format(project_name)
+ items = []
+ items.append({
+ "type": "label",
+ "value": "# {}".format(msg)
+ })
+ items.append({
+ "type": "label",
+ "value": (
+ "Download report from job for more information.
"
+ )
+ })
+
+ report = {}
+ try:
+ report = self.entities_factory.report()
+ except Exception:
+ pass
+
+ _items = report.get("items") or []
+ if _items:
+ items.append(self.entities_factory.report_splitter)
+ items.extend(_items)
+
+ self.show_interface(items, title, event, submit_btn_label="Ok")
+
+ return {"success": True, "message": msg}
+
+ job_entity["status"] = "done"
+ job_entity["data"] = json.dumps({
+ "description": "Sync to avalon finished."
+ })
+ session.commit()
+
+ return result
+
+ def synchronization(self, event, project_name):
+ time_start = time.time()
+
+ self.show_message(event, "Synchronization - Preparing data", True)
+
+ try:
+ output = self.entities_factory.launch_setup(project_name)
+ if output is not None:
+ return output
+
+ time_1 = time.time()
+
+ self.entities_factory.set_cutom_attributes()
+ time_2 = time.time()
+
+ # This must happen before all filtering!!!
+ self.entities_factory.prepare_avalon_entities(project_name)
+ time_3 = time.time()
+
+ self.entities_factory.filter_by_ignore_sync()
+ time_4 = time.time()
+
+ self.entities_factory.duplicity_regex_check()
+ time_5 = time.time()
+
+ self.entities_factory.prepare_ftrack_ent_data()
+ time_6 = time.time()
+
+ self.entities_factory.synchronize()
+ time_7 = time.time()
+
+ self.log.debug(
+ "*** Synchronization finished ***"
+ )
+ self.log.debug(
+ "preparation <{}>".format(time_1 - time_start)
+ )
+ self.log.debug(
+ "set_cutom_attributes <{}>".format(time_2 - time_1)
+ )
+ self.log.debug(
+ "prepare_avalon_entities <{}>".format(time_3 - time_2)
+ )
+ self.log.debug(
+ "filter_by_ignore_sync <{}>".format(time_4 - time_3)
+ )
+ self.log.debug(
+ "duplicity_regex_check <{}>".format(time_5 - time_4)
+ )
+ self.log.debug(
+ "prepare_ftrack_ent_data <{}>".format(time_6 - time_5)
+ )
+ self.log.debug(
+ "synchronize <{}>".format(time_7 - time_6)
+ )
+ self.log.debug(
+ "* Total time: {}".format(time_7 - time_start)
+ )
+
+ if self.entities_factory.project_created:
+ event = ftrack_api.event.base.Event(
+ topic="openpype.project.created",
+ data={"project_name": project_name}
+ )
+ self.session.event_hub.publish(event)
+
+ report = self.entities_factory.report()
+ if report and report.get("items"):
+ default_title = "Synchronization report ({}):".format(
+ project_name
+ )
+ self.show_interface(
+ items=report["items"],
+ title=report.get("title", default_title),
+ event=event
+ )
+ return {
+ "success": True,
+ "message": "Synchronization Finished"
+ }
+
+ finally:
+ try:
+ self.entities_factory.dbcon.uninstall()
+ except Exception:
+ pass
+
+ try:
+ self.entities_factory.session.close()
+ except Exception:
+ pass
+
+
+def register(session):
+ '''Register plugin. Called when used as an plugin.'''
+
+ SyncToAvalonLocal(session).register()
diff --git a/openpype/modules/default_modules/ftrack/event_handlers_user/action_test.py b/openpype/modules/ftrack/event_handlers_user/action_test.py
similarity index 100%
rename from openpype/modules/default_modules/ftrack/event_handlers_user/action_test.py
rename to openpype/modules/ftrack/event_handlers_user/action_test.py
diff --git a/openpype/modules/default_modules/ftrack/event_handlers_user/action_thumbnail_to_childern.py b/openpype/modules/ftrack/event_handlers_user/action_thumbnail_to_childern.py
similarity index 100%
rename from openpype/modules/default_modules/ftrack/event_handlers_user/action_thumbnail_to_childern.py
rename to openpype/modules/ftrack/event_handlers_user/action_thumbnail_to_childern.py
diff --git a/openpype/modules/default_modules/ftrack/event_handlers_user/action_thumbnail_to_parent.py b/openpype/modules/ftrack/event_handlers_user/action_thumbnail_to_parent.py
similarity index 100%
rename from openpype/modules/default_modules/ftrack/event_handlers_user/action_thumbnail_to_parent.py
rename to openpype/modules/ftrack/event_handlers_user/action_thumbnail_to_parent.py
diff --git a/openpype/modules/default_modules/ftrack/event_handlers_user/action_where_run_ask.py b/openpype/modules/ftrack/event_handlers_user/action_where_run_ask.py
similarity index 100%
rename from openpype/modules/default_modules/ftrack/event_handlers_user/action_where_run_ask.py
rename to openpype/modules/ftrack/event_handlers_user/action_where_run_ask.py
diff --git a/openpype/modules/default_modules/ftrack/ftrack_module.py b/openpype/modules/ftrack/ftrack_module.py
similarity index 80%
rename from openpype/modules/default_modules/ftrack/ftrack_module.py
rename to openpype/modules/ftrack/ftrack_module.py
index 8a7525d65b1..75ffd7f8640 100644
--- a/openpype/modules/default_modules/ftrack/ftrack_module.py
+++ b/openpype/modules/ftrack/ftrack_module.py
@@ -1,26 +1,27 @@
import os
import json
import collections
+import platform
import click
from openpype.modules import OpenPypeModule
-from openpype_interfaces import (
+from openpype.modules.interfaces import (
ITrayModule,
IPluginPaths,
- ILaunchHookPaths,
ISettingsChangeListener
)
from openpype.settings import SaveWarningExc
+from openpype.lib import Logger
FTRACK_MODULE_DIR = os.path.dirname(os.path.abspath(__file__))
+_URL_NOT_SET = object()
class FtrackModule(
OpenPypeModule,
ITrayModule,
IPluginPaths,
- ILaunchHookPaths,
ISettingsChangeListener
):
name = "ftrack"
@@ -29,31 +30,30 @@ def initialize(self, settings):
ftrack_settings = settings[self.name]
self.enabled = ftrack_settings["enabled"]
- # Add http schema
- ftrack_url = ftrack_settings["ftrack_server"].strip("/ ")
- if ftrack_url:
- if "http" not in ftrack_url:
- ftrack_url = "https://" + ftrack_url
-
- # Check if "ftrack.app" is part os url
- if "ftrackapp.com" not in ftrack_url:
- ftrack_url = ftrack_url + ".ftrackapp.com"
-
- self.ftrack_url = ftrack_url
+ self._settings_ftrack_url = ftrack_settings["ftrack_server"]
+ self._ftrack_url = _URL_NOT_SET
current_dir = os.path.dirname(os.path.abspath(__file__))
+ low_platform = platform.system().lower()
+
+ # Server event handler paths
server_event_handlers_paths = [
os.path.join(current_dir, "event_handlers_server")
]
- server_event_handlers_paths.extend(
- ftrack_settings["ftrack_events_path"]
- )
+ settings_server_paths = ftrack_settings["ftrack_events_path"]
+ if isinstance(settings_server_paths, dict):
+ settings_server_paths = settings_server_paths[low_platform]
+ server_event_handlers_paths.extend(settings_server_paths)
+
+ # User event handler paths
user_event_handlers_paths = [
os.path.join(current_dir, "event_handlers_user")
]
- user_event_handlers_paths.extend(
- ftrack_settings["ftrack_actions_path"]
- )
+ settings_action_paths = ftrack_settings["ftrack_actions_path"]
+ if isinstance(settings_action_paths, dict):
+ settings_action_paths = settings_action_paths[low_platform]
+ user_event_handlers_paths.extend(settings_action_paths)
+
# Prepare attribute
self.server_event_handlers_paths = server_event_handlers_paths
self.user_event_handlers_paths = user_event_handlers_paths
@@ -63,6 +63,16 @@ def initialize(self, settings):
self.timers_manager_connector = None
self._timers_manager_module = None
+ def get_ftrack_url(self):
+ if self._ftrack_url is _URL_NOT_SET:
+ self._ftrack_url = resolve_ftrack_url(
+ self._settings_ftrack_url,
+ logger=self.log
+ )
+ return self._ftrack_url
+
+ ftrack_url = property(get_ftrack_url)
+
def get_global_environments(self):
"""Ftrack's global environments."""
return {
@@ -76,9 +86,44 @@ def get_plugin_paths(self):
}
def get_launch_hook_paths(self):
- """Implementation of `ILaunchHookPaths`."""
+ """Implementation for applications launch hooks."""
+
return os.path.join(FTRACK_MODULE_DIR, "launch_hooks")
+ def modify_application_launch_arguments(self, application, env):
+ if not application.use_python_2:
+ return
+
+ self.log.info("Adding Ftrack Python 2 packages to PYTHONPATH.")
+
+ # Prepare vendor dir path
+ python_2_vendor = os.path.join(FTRACK_MODULE_DIR, "python2_vendor")
+
+ # Add Python 2 modules
+ python_paths = [
+ # `python-ftrack-api`
+ os.path.join(python_2_vendor, "ftrack-python-api", "source"),
+ # `arrow`
+ os.path.join(python_2_vendor, "arrow"),
+ # `builtins` from `python-future`
+ # - `python-future` is strict Python 2 module that cause crashes
+ # of Python 3 scripts executed through OpenPype
+ # (burnin script etc.)
+ os.path.join(python_2_vendor, "builtins"),
+ # `backports.functools_lru_cache`
+ os.path.join(
+ python_2_vendor, "backports.functools_lru_cache"
+ )
+ ]
+
+ # Load PYTHONPATH from current launch context
+ python_path = env.get("PYTHONPATH")
+ if python_path:
+ python_paths.append(python_path)
+
+ # Set new PYTHONPATH to launch context environments
+ env["PYTHONPATH"] = os.pathsep.join(python_paths)
+
def connect_with_modules(self, enabled_modules):
for module in enabled_modules:
if not hasattr(module, "get_ftrack_event_handler_paths"):
@@ -294,9 +339,10 @@ def on_project_anatomy_save(
# TODO add add permissions check
# TODO add value validations
# - value type and list items
- entity_key = collections.OrderedDict()
- entity_key["configuration_id"] = configuration["id"]
- entity_key["entity_id"] = project_id
+ entity_key = collections.OrderedDict([
+ ("configuration_id", configuration["id"]),
+ ("entity_id", project_id)
+ ])
session.recorded_operations.push(
ftrack_api.operation.UpdateEntityOperation(
@@ -436,6 +482,51 @@ def cli(self, click_group):
click_group.add_command(cli_main)
+def _check_ftrack_url(url):
+ import requests
+
+ try:
+ result = requests.get(url, allow_redirects=False)
+ except requests.exceptions.RequestException:
+ return False
+
+ if (result.status_code != 200 or "FTRACK_VERSION" not in result.headers):
+ return False
+ return True
+
+
+def resolve_ftrack_url(url, logger=None):
+ """Checks if Ftrack server is responding."""
+
+ if logger is None:
+ logger = Logger.get_logger(__name__)
+
+ url = url.strip("/ ")
+ if not url:
+ logger.error("Ftrack URL is not set!")
+ return None
+
+ if not url.startswith("http"):
+ url = "https://" + url
+
+ ftrack_url = None
+ if not url.endswith("ftrackapp.com"):
+ ftrackapp_url = url + ".ftrackapp.com"
+ if _check_ftrack_url(ftrackapp_url):
+ ftrack_url = ftrackapp_url
+
+ if not ftrack_url and _check_ftrack_url(url):
+ ftrack_url = url
+
+ if ftrack_url:
+ logger.debug("Ftrack server \"{}\" is accessible.".format(ftrack_url))
+
+ else:
+ logger.error("Ftrack server \"{}\" is not accessible!".format(url))
+
+ return ftrack_url
+
+
@click.group(FtrackModule.name, help="Ftrack module related commands.")
def cli_main():
pass
diff --git a/openpype/modules/ftrack/ftrack_server/__init__.py b/openpype/modules/ftrack/ftrack_server/__init__.py
new file mode 100644
index 00000000000..8e5f7c4c518
--- /dev/null
+++ b/openpype/modules/ftrack/ftrack_server/__init__.py
@@ -0,0 +1,6 @@
+from .ftrack_server import FtrackServer
+
+
+__all__ = (
+ "FtrackServer",
+)
diff --git a/openpype/modules/default_modules/ftrack/ftrack_server/event_server_cli.py b/openpype/modules/ftrack/ftrack_server/event_server_cli.py
similarity index 96%
rename from openpype/modules/default_modules/ftrack/ftrack_server/event_server_cli.py
rename to openpype/modules/ftrack/ftrack_server/event_server_cli.py
index 1a76905b385..20c5ab24a8b 100644
--- a/openpype/modules/default_modules/ftrack/ftrack_server/event_server_cli.py
+++ b/openpype/modules/ftrack/ftrack_server/event_server_cli.py
@@ -1,11 +1,9 @@
import os
-import sys
import signal
import datetime
import subprocess
import socket
import json
-import platform
import getpass
import atexit
import time
@@ -13,16 +11,20 @@
import ftrack_api
import pymongo
-from openpype.lib import (
- get_pype_execute_args,
+from openpype.client.mongo import (
OpenPypeMongoConnection,
+ validate_mongo_connection,
+)
+from openpype.lib import (
+ get_openpype_execute_args,
get_openpype_version,
get_build_version,
- validate_mongo_connection
)
-from openpype_modules.ftrack import FTRACK_MODULE_DIR
+from openpype_modules.ftrack import (
+ FTRACK_MODULE_DIR,
+ resolve_ftrack_url,
+)
from openpype_modules.ftrack.lib import credentials
-from openpype_modules.ftrack.ftrack_server.lib import check_ftrack_url
from openpype_modules.ftrack.ftrack_server import socket_thread
@@ -114,7 +116,7 @@ def legacy_server(ftrack_url):
while True:
if not ftrack_accessible:
- ftrack_accessible = check_ftrack_url(ftrack_url)
+ ftrack_accessible = resolve_ftrack_url(ftrack_url)
# Run threads only if Ftrack is accessible
if not ftrack_accessible and not printed_ftrack_error:
@@ -136,7 +138,7 @@ def legacy_server(ftrack_url):
if subproc is None:
if subproc_failed_count < max_fail_count:
- args = get_pype_execute_args("run", subproc_path)
+ args = get_openpype_execute_args("run", subproc_path)
subproc = subprocess.Popen(
args,
stdout=subprocess.PIPE
@@ -248,7 +250,7 @@ def on_exit(processor_thread, storer_thread, statuser_thread):
["Username", getpass.getuser()],
["Host Name", host_name],
["Host IP", socket.gethostbyname(host_name)],
- ["OpenPype executable", get_pype_execute_args()[-1]],
+ ["OpenPype executable", get_openpype_execute_args()[-1]],
["OpenPype version", get_openpype_version() or "N/A"],
["OpenPype build version", get_build_version() or "N/A"]
]
@@ -257,7 +259,7 @@ def on_exit(processor_thread, storer_thread, statuser_thread):
while True:
# Check if accessible Ftrack and Mongo url
if not ftrack_accessible:
- ftrack_accessible = check_ftrack_url(ftrack_url)
+ ftrack_accessible = resolve_ftrack_url(ftrack_url)
if not mongo_accessible:
mongo_accessible = check_mongo_url(mongo_uri)
@@ -441,7 +443,7 @@ def run_event_server(
os.environ["CLOCKIFY_API_KEY"] = clockify_api_key
# Check url regex and accessibility
- ftrack_url = check_ftrack_url(ftrack_url)
+ ftrack_url = resolve_ftrack_url(ftrack_url)
if not ftrack_url:
print('Exiting! < Please enter Ftrack server url >')
return 1
diff --git a/openpype/modules/default_modules/ftrack/ftrack_server/ftrack_server.py b/openpype/modules/ftrack/ftrack_server/ftrack_server.py
similarity index 89%
rename from openpype/modules/default_modules/ftrack/ftrack_server/ftrack_server.py
rename to openpype/modules/ftrack/ftrack_server/ftrack_server.py
index bd67fba3d6e..c75b8f71724 100644
--- a/openpype/modules/default_modules/ftrack/ftrack_server/ftrack_server.py
+++ b/openpype/modules/ftrack/ftrack_server/ftrack_server.py
@@ -7,12 +7,10 @@
import ftrack_api
from openpype.lib import (
- PypeLogger,
+ Logger,
modules_from_path
)
-log = PypeLogger.get_logger(__name__)
-
"""
# Required - Needed for connection to Ftrack
FTRACK_SERVER # Ftrack server e.g. "https://myFtrack.ftrackapp.com"
@@ -43,10 +41,13 @@ def __init__(self, handler_paths=None):
server.run_server()
..
"""
+
# set Ftrack logging to Warning only - OPTIONAL
ftrack_log = logging.getLogger("ftrack_api")
ftrack_log.setLevel(logging.WARNING)
+ self.log = Logger.get_logger(__name__)
+
self.stopped = True
self.is_running = False
@@ -63,10 +64,16 @@ def set_files(self, paths):
# Iterate all paths
register_functions = []
for path in paths:
+ # Try to format path with environments
+ try:
+ path = path.format(**os.environ)
+ except BaseException:
+ pass
+
# Get all modules with functions
modules, crashed = modules_from_path(path)
for filepath, exc_info in crashed:
- log.warning("Filepath load crashed {}.\n{}".format(
+ self.log.warning("Filepath load crashed {}.\n{}".format(
filepath, traceback.format_exception(*exc_info)
))
@@ -81,7 +88,7 @@ def set_files(self, paths):
break
if not register_function:
- log.warning(
+ self.log.warning(
"\"{}\" - Missing register method".format(filepath)
)
continue
@@ -91,7 +98,7 @@ def set_files(self, paths):
)
if not register_functions:
- log.warning((
+ self.log.warning((
"There are no events with `register` function"
" in registered paths: \"{}\""
).format("| ".join(paths)))
@@ -100,7 +107,7 @@ def set_files(self, paths):
try:
register_func(self.session)
except Exception:
- log.warning(
+ self.log.warning(
"\"{}\" - register was not successful".format(filepath),
exc_info=True
)
@@ -135,7 +142,7 @@ def run_server(self, session=None, load_files=True):
self.session = session
if load_files:
if not self.handler_paths:
- log.warning((
+ self.log.warning((
"Paths to event handlers are not set."
" Ftrack server won't launch."
))
@@ -145,8 +152,8 @@ def run_server(self, session=None, load_files=True):
self.set_files(self.handler_paths)
msg = "Registration of event handlers has finished!"
- log.info(len(msg) * "*")
- log.info(msg)
+ self.log.info(len(msg) * "*")
+ self.log.info(msg)
# keep event_hub on session running
self.session.event_hub.wait()
diff --git a/openpype/modules/ftrack/ftrack_server/lib.py b/openpype/modules/ftrack/ftrack_server/lib.py
new file mode 100644
index 00000000000..c8143f739ca
--- /dev/null
+++ b/openpype/modules/ftrack/ftrack_server/lib.py
@@ -0,0 +1,397 @@
+import os
+import sys
+import logging
+import getpass
+import atexit
+import threading
+import datetime
+import time
+import queue
+import collections
+import appdirs
+import pymongo
+
+import requests
+import ftrack_api
+import ftrack_api.session
+import ftrack_api.cache
+import ftrack_api.operation
+import ftrack_api._centralized_storage_scenario
+import ftrack_api.event
+from ftrack_api.logging import LazyLogMessage as L
+try:
+ from weakref import WeakMethod
+except ImportError:
+ from ftrack_api._weakref import WeakMethod
+from openpype_modules.ftrack.lib import get_ftrack_event_mongo_info
+
+from openpype.client import OpenPypeMongoConnection
+from openpype.lib import Logger
+
+TOPIC_STATUS_SERVER = "openpype.event.server.status"
+TOPIC_STATUS_SERVER_RESULT = "openpype.event.server.status.result"
+
+
+class SocketBaseEventHub(ftrack_api.event.hub.EventHub):
+
+ hearbeat_msg = b"hearbeat"
+ heartbeat_callbacks = []
+
+ def __init__(self, *args, **kwargs):
+ self.sock = kwargs.pop("sock")
+ super(SocketBaseEventHub, self).__init__(*args, **kwargs)
+
+ def _handle_packet(self, code, packet_identifier, path, data):
+ """Override `_handle_packet` which extend heartbeat"""
+ code_name = self._code_name_mapping[code]
+ if code_name == "heartbeat":
+ # Reply with heartbeat.
+ for callback in self.heartbeat_callbacks:
+ callback()
+
+ self.sock.sendall(self.hearbeat_msg)
+ return self._send_packet(self._code_name_mapping["heartbeat"])
+
+ return super(SocketBaseEventHub, self)._handle_packet(
+ code, packet_identifier, path, data
+ )
+
+
+class StatusEventHub(SocketBaseEventHub):
+ def _handle_packet(self, code, packet_identifier, path, data):
+ """Override `_handle_packet` which extend heartbeat"""
+ code_name = self._code_name_mapping[code]
+ if code_name == "connect":
+ event = ftrack_api.event.base.Event(
+ topic="openpype.status.started",
+ data={},
+ source={
+ "id": self.id,
+ "user": {"username": self._api_user}
+ }
+ )
+ self._event_queue.put(event)
+
+ return super(StatusEventHub, self)._handle_packet(
+ code, packet_identifier, path, data
+ )
+
+
+class StorerEventHub(SocketBaseEventHub):
+
+ hearbeat_msg = b"storer"
+
+ def _handle_packet(self, code, packet_identifier, path, data):
+ """Override `_handle_packet` which extend heartbeat"""
+ code_name = self._code_name_mapping[code]
+ if code_name == "connect":
+ event = ftrack_api.event.base.Event(
+ topic="openpype.storer.started",
+ data={},
+ source={
+ "id": self.id,
+ "user": {"username": self._api_user}
+ }
+ )
+ self._event_queue.put(event)
+
+ return super(StorerEventHub, self)._handle_packet(
+ code, packet_identifier, path, data
+ )
+
+
+class ProcessEventHub(SocketBaseEventHub):
+ hearbeat_msg = b"processor"
+
+ is_collection_created = False
+ pypelog = Logger.get_logger("Session Processor")
+
+ def __init__(self, *args, **kwargs):
+ self.mongo_url = None
+ self.dbcon = None
+
+ super(ProcessEventHub, self).__init__(*args, **kwargs)
+
+ def prepare_dbcon(self):
+ try:
+ database_name, collection_name = get_ftrack_event_mongo_info()
+ mongo_client = OpenPypeMongoConnection.get_mongo_client()
+ self.dbcon = mongo_client[database_name][collection_name]
+ self.mongo_client = mongo_client
+
+ except pymongo.errors.AutoReconnect:
+ self.pypelog.error((
+ "Mongo server \"{}\" is not responding, exiting."
+ ).format(OpenPypeMongoConnection.get_default_mongo_url()))
+ sys.exit(0)
+
+ except pymongo.errors.OperationFailure:
+ self.pypelog.error((
+ "Error with Mongo access, probably permissions."
+ "Check if exist database with name \"{}\""
+ " and collection \"{}\" inside."
+ ).format(self.database, self.collection_name))
+ self.sock.sendall(b"MongoError")
+ sys.exit(0)
+
+ def wait(self, duration=None):
+ """Overridden wait
+ Event are loaded from Mongo DB when queue is empty. Handled event is
+ set as processed in Mongo DB.
+ """
+ started = time.time()
+ self.prepare_dbcon()
+ while True:
+ try:
+ event = self._event_queue.get(timeout=0.1)
+ except queue.Empty:
+ if not self.load_events():
+ time.sleep(0.5)
+ else:
+ try:
+ self._handle(event)
+
+ mongo_id = event["data"].get("_event_mongo_id")
+ if mongo_id is None:
+ continue
+
+ self.dbcon.update_one(
+ {"_id": mongo_id},
+ {"$set": {"pype_data.is_processed": True}}
+ )
+
+ except pymongo.errors.AutoReconnect:
+ self.pypelog.error((
+ "Mongo server \"{}\" is not responding, exiting."
+ ).format(os.environ["OPENPYPE_MONGO"]))
+ sys.exit(0)
+ # Additional special processing of events.
+ if event['topic'] == 'ftrack.meta.disconnected':
+ break
+
+ if duration is not None:
+ if (time.time() - started) > duration:
+ break
+
+ def load_events(self):
+ """Load not processed events sorted by stored date"""
+ ago_date = datetime.datetime.now() - datetime.timedelta(days=3)
+ self.dbcon.delete_many({
+ "pype_data.stored": {"$lte": ago_date},
+ "pype_data.is_processed": True
+ })
+
+ not_processed_events = self.dbcon.find(
+ {"pype_data.is_processed": False}
+ ).sort(
+ [("pype_data.stored", pymongo.ASCENDING)]
+ )
+
+ found = False
+ for event_data in not_processed_events:
+ new_event_data = {
+ k: v for k, v in event_data.items()
+ if k not in ["_id", "pype_data"]
+ }
+ try:
+ event = ftrack_api.event.base.Event(**new_event_data)
+ event["data"]["_event_mongo_id"] = event_data["_id"]
+ except Exception:
+ self.logger.exception(L(
+ 'Failed to convert payload into event: {0}',
+ event_data
+ ))
+ continue
+ found = True
+ self._event_queue.put(event)
+
+ return found
+
+ def _handle_packet(self, code, packet_identifier, path, data):
+ """Override `_handle_packet` which skip events and extend heartbeat"""
+ code_name = self._code_name_mapping[code]
+ if code_name == "event":
+ return
+
+ return super()._handle_packet(code, packet_identifier, path, data)
+
+
+class CustomEventHubSession(ftrack_api.session.Session):
+ '''An isolated session for interaction with an ftrack server.'''
+ def __init__(
+ self, server_url=None, api_key=None, api_user=None, auto_populate=True,
+ plugin_paths=None, cache=None, cache_key_maker=None,
+ auto_connect_event_hub=False, schema_cache_path=None,
+ plugin_arguments=None, timeout=60, **kwargs
+ ):
+ self.kwargs = kwargs
+
+ super(ftrack_api.session.Session, self).__init__()
+ self.logger = logging.getLogger(
+ __name__ + '.' + self.__class__.__name__
+ )
+ self._closed = False
+
+ if server_url is None:
+ server_url = os.environ.get('FTRACK_SERVER')
+
+ if not server_url:
+ raise TypeError(
+ 'Required "server_url" not specified. Pass as argument or set '
+ 'in environment variable FTRACK_SERVER.'
+ )
+
+ self._server_url = server_url
+
+ if api_key is None:
+ api_key = os.environ.get(
+ 'FTRACK_API_KEY',
+ # Backwards compatibility
+ os.environ.get('FTRACK_APIKEY')
+ )
+
+ if not api_key:
+ raise TypeError(
+ 'Required "api_key" not specified. Pass as argument or set in '
+ 'environment variable FTRACK_API_KEY.'
+ )
+
+ self._api_key = api_key
+
+ if api_user is None:
+ api_user = os.environ.get('FTRACK_API_USER')
+ if not api_user:
+ try:
+ api_user = getpass.getuser()
+ except Exception:
+ pass
+
+ if not api_user:
+ raise TypeError(
+ 'Required "api_user" not specified. Pass as argument, set in '
+ 'environment variable FTRACK_API_USER or one of the standard '
+ 'environment variables used by Python\'s getpass module.'
+ )
+
+ self._api_user = api_user
+
+ # Currently pending operations.
+ self.recorded_operations = ftrack_api.operation.Operations()
+
+ # OpenPype change - In new API are operations properties
+ new_api = hasattr(self.__class__, "record_operations")
+
+ if new_api:
+ self._record_operations = collections.defaultdict(
+ lambda: True
+ )
+ self._auto_populate = collections.defaultdict(
+ lambda: auto_populate
+ )
+ else:
+ self.record_operations = True
+ self.auto_populate = auto_populate
+
+ self.cache_key_maker = cache_key_maker
+ if self.cache_key_maker is None:
+ self.cache_key_maker = ftrack_api.cache.StringKeyMaker()
+
+ # Enforce always having a memory cache at top level so that the same
+ # in-memory instance is returned from session.
+ self.cache = ftrack_api.cache.LayeredCache([
+ ftrack_api.cache.MemoryCache()
+ ])
+
+ if cache is not None:
+ if callable(cache):
+ cache = cache(self)
+
+ if cache is not None:
+ self.cache.caches.append(cache)
+
+ if new_api:
+ self.merge_lock = threading.RLock()
+
+ self._managed_request = None
+ self._request = requests.Session()
+ self._request.auth = ftrack_api.session.SessionAuthentication(
+ self._api_key, self._api_user
+ )
+ self.request_timeout = timeout
+
+ # Fetch server information and in doing so also check credentials.
+ self._server_information = self._fetch_server_information()
+
+ # Now check compatibility of server based on retrieved information.
+ self.check_server_compatibility()
+
+ # Construct event hub and load plugins.
+ self._event_hub = self._create_event_hub()
+
+ self._auto_connect_event_hub_thread = None
+ if auto_connect_event_hub:
+ # Connect to event hub in background thread so as not to block main
+ # session usage waiting for event hub connection.
+ self._auto_connect_event_hub_thread = threading.Thread(
+ target=self._event_hub.connect
+ )
+ self._auto_connect_event_hub_thread.daemon = True
+ self._auto_connect_event_hub_thread.start()
+
+ # Register to auto-close session on exit.
+ atexit.register(WeakMethod(self.close))
+
+ self._plugin_paths = plugin_paths
+ if self._plugin_paths is None:
+ self._plugin_paths = os.environ.get(
+ 'FTRACK_EVENT_PLUGIN_PATH', ''
+ ).split(os.pathsep)
+
+ self._discover_plugins(plugin_arguments=plugin_arguments)
+
+ # TODO: Make schemas read-only and non-mutable (or at least without
+ # rebuilding types)?
+ if schema_cache_path is not False:
+ if schema_cache_path is None:
+ schema_cache_path = appdirs.user_cache_dir()
+ schema_cache_path = os.environ.get(
+ 'FTRACK_API_SCHEMA_CACHE_PATH', schema_cache_path
+ )
+
+ schema_cache_path = os.path.join(
+ schema_cache_path, 'ftrack_api_schema_cache.json'
+ )
+
+ self.schemas = self._load_schemas(schema_cache_path)
+ self.types = self._build_entity_type_classes(self.schemas)
+
+ ftrack_api._centralized_storage_scenario.register(self)
+
+ self._configure_locations()
+ self.event_hub.publish(
+ ftrack_api.event.base.Event(
+ topic='ftrack.api.session.ready',
+ data=dict(
+ session=self
+ )
+ ),
+ synchronous=True
+ )
+
+ def _create_event_hub(self):
+ return ftrack_api.event.hub.EventHub(
+ self._server_url,
+ self._api_user,
+ self._api_key
+ )
+
+
+class SocketSession(CustomEventHubSession):
+ def _create_event_hub(self):
+ self.sock = self.kwargs["sock"]
+ return self.kwargs["Eventhub"](
+ self._server_url,
+ self._api_user,
+ self._api_key,
+ sock=self.sock
+ )
diff --git a/openpype/modules/default_modules/ftrack/ftrack_server/socket_thread.py b/openpype/modules/ftrack/ftrack_server/socket_thread.py
similarity index 97%
rename from openpype/modules/default_modules/ftrack/ftrack_server/socket_thread.py
rename to openpype/modules/ftrack/ftrack_server/socket_thread.py
index eb8ec4d06c8..3ef55f8daae 100644
--- a/openpype/modules/default_modules/ftrack/ftrack_server/socket_thread.py
+++ b/openpype/modules/ftrack/ftrack_server/socket_thread.py
@@ -5,8 +5,8 @@
import threading
import traceback
import subprocess
-from openpype.api import Logger
-from openpype.lib import get_pype_execute_args
+
+from openpype.lib import get_openpype_execute_args, Logger
class SocketThread(threading.Thread):
@@ -16,7 +16,7 @@ class SocketThread(threading.Thread):
def __init__(self, name, port, filepath, additional_args=[]):
super(SocketThread, self).__init__()
- self.log = Logger().get_logger(self.__class__.__name__)
+ self.log = Logger.get_logger(self.__class__.__name__)
self.setName(name)
self.name = name
self.port = port
@@ -59,7 +59,7 @@ def run(self):
env = os.environ.copy()
env["OPENPYPE_PROCESS_MONGO_ID"] = str(Logger.mongo_process_id)
# OpenPype executable (with path to start script if not build)
- args = get_pype_execute_args(
+ args = get_openpype_execute_args(
# Add `run` command
"run",
self.filepath,
diff --git a/openpype/modules/default_modules/ftrack/launch_hooks/post_ftrack_changes.py b/openpype/modules/ftrack/launch_hooks/post_ftrack_changes.py
similarity index 100%
rename from openpype/modules/default_modules/ftrack/launch_hooks/post_ftrack_changes.py
rename to openpype/modules/ftrack/launch_hooks/post_ftrack_changes.py
diff --git a/openpype/modules/ftrack/lib/__init__.py b/openpype/modules/ftrack/lib/__init__.py
new file mode 100644
index 00000000000..7fc2bc99eb5
--- /dev/null
+++ b/openpype/modules/ftrack/lib/__init__.py
@@ -0,0 +1,56 @@
+from .constants import (
+ CUST_ATTR_ID_KEY,
+ CUST_ATTR_AUTO_SYNC,
+ CUST_ATTR_GROUP,
+ CUST_ATTR_TOOLS,
+ CUST_ATTR_APPLICATIONS,
+ CUST_ATTR_INTENT,
+ FPS_KEYS
+)
+from .settings import (
+ get_ftrack_event_mongo_info
+)
+from .custom_attributes import (
+ default_custom_attributes_definition,
+ app_definitions_from_app_manager,
+ tool_definitions_from_app_manager,
+ get_openpype_attr,
+ query_custom_attributes
+)
+
+from . import avalon_sync
+from . import credentials
+from .ftrack_base_handler import BaseHandler
+from .ftrack_event_handler import BaseEvent
+from .ftrack_action_handler import BaseAction, ServerAction, statics_icon
+
+
+__all__ = (
+ "CUST_ATTR_ID_KEY",
+ "CUST_ATTR_AUTO_SYNC",
+ "CUST_ATTR_GROUP",
+ "CUST_ATTR_TOOLS",
+ "CUST_ATTR_APPLICATIONS",
+ "CUST_ATTR_INTENT",
+ "FPS_KEYS",
+
+ "get_ftrack_event_mongo_info",
+
+ "default_custom_attributes_definition",
+ "app_definitions_from_app_manager",
+ "tool_definitions_from_app_manager",
+ "get_openpype_attr",
+ "query_custom_attributes",
+
+ "avalon_sync",
+
+ "credentials",
+
+ "BaseHandler",
+
+ "BaseEvent",
+
+ "BaseAction",
+ "ServerAction",
+ "statics_icon"
+)
diff --git a/openpype/modules/default_modules/ftrack/lib/avalon_sync.py b/openpype/modules/ftrack/lib/avalon_sync.py
similarity index 92%
rename from openpype/modules/default_modules/ftrack/lib/avalon_sync.py
rename to openpype/modules/ftrack/lib/avalon_sync.py
index f58eb914853..935d1e85c9d 100644
--- a/openpype/modules/default_modules/ftrack/lib/avalon_sync.py
+++ b/openpype/modules/ftrack/lib/avalon_sync.py
@@ -1,24 +1,30 @@
-import os
import re
import json
import collections
import copy
+import numbers
import six
-from avalon.api import AvalonMongoDB
-
-import avalon
-
-from openpype.api import (
- Logger,
- Anatomy,
- get_anatomy_settings
+from openpype.client import (
+ get_project,
+ get_assets,
+ get_archived_assets,
+ get_subsets,
+ get_versions,
+ get_representations
+)
+from openpype.client.operations import (
+ CURRENT_ASSET_DOC_SCHEMA,
+ CURRENT_PROJECT_SCHEMA,
+ CURRENT_PROJECT_CONFIG_SCHEMA,
)
-from openpype.lib import ApplicationManager
+from openpype.settings import get_anatomy_settings
+from openpype.lib import ApplicationManager, Logger
+from openpype.pipeline import AvalonMongoDB, schema
-from .constants import CUST_ATTR_ID_KEY
-from .custom_attributes import get_openpype_attr
+from .constants import CUST_ATTR_ID_KEY, FPS_KEYS
+from .custom_attributes import get_openpype_attr, query_custom_attributes
from bson.objectid import ObjectId
from bson.errors import InvalidId
@@ -28,12 +34,131 @@
log = Logger.get_logger(__name__)
-# Current schemas for avalon types
-CURRENT_DOC_SCHEMAS = {
- "project": "openpype:project-3.0",
- "asset": "openpype:asset-3.0",
- "config": "openpype:config-2.0"
-}
+class InvalidFpsValue(Exception):
+ pass
+
+
+def is_string_number(value):
+ """Can string value be converted to number (float)."""
+ if not isinstance(value, six.string_types):
+ raise TypeError("Expected {} got {}".format(
+ ", ".join(str(t) for t in six.string_types), str(type(value))
+ ))
+ if value == ".":
+ return False
+
+ if value.startswith("."):
+ value = "0" + value
+ elif value.endswith("."):
+ value = value + "0"
+
+ if re.match(r"^\d+(\.\d+)?$", value) is None:
+ return False
+ return True
+
+
+def convert_to_fps(source_value):
+ """Convert value into fps value.
+
+ Non string values are kept untouched. String is tried to convert.
+ Valid values:
+ "1000"
+ "1000.05"
+ "1000,05"
+ ",05"
+ ".05"
+ "1000,"
+ "1000."
+ "1000/1000"
+ "1000.05/1000"
+ "1000/1000.05"
+ "1000.05/1000.05"
+ "1000,05/1000"
+ "1000/1000,05"
+ "1000,05/1000,05"
+
+ Invalid values:
+ "/"
+ "/1000"
+ "1000/"
+ ","
+ "."
+ ...any other string
+
+ Returns:
+ float: Converted value.
+
+ Raises:
+ InvalidFpsValue: When value can't be converted to float.
+ """
+ if not isinstance(source_value, six.string_types):
+ if isinstance(source_value, numbers.Number):
+ return float(source_value)
+ return source_value
+
+ value = source_value.strip().replace(",", ".")
+ if not value:
+ raise InvalidFpsValue("Got empty value")
+
+ subs = value.split("/")
+ if len(subs) == 1:
+ str_value = subs[0]
+ if not is_string_number(str_value):
+ raise InvalidFpsValue(
+ "Value \"{}\" can't be converted to number.".format(value)
+ )
+ return float(str_value)
+
+ elif len(subs) == 2:
+ divident, divisor = subs
+ if not divident or not is_string_number(divident):
+ raise InvalidFpsValue(
+ "Divident value \"{}\" can't be converted to number".format(
+ divident
+ )
+ )
+
+ if not divisor or not is_string_number(divisor):
+ raise InvalidFpsValue(
+ "Divisor value \"{}\" can't be converted to number".format(
+ divident
+ )
+ )
+ divisor_float = float(divisor)
+ if divisor_float == 0.0:
+ raise InvalidFpsValue("Can't divide by zero")
+ return float(divident) / divisor_float
+
+ raise InvalidFpsValue(
+ "Value can't be converted to number \"{}\"".format(source_value)
+ )
+
+
+def create_chunks(iterable, chunk_size=None):
+ """Separate iterable into multiple chunks by size.
+
+ Args:
+ iterable(list|tuple|set): Object that will be separated into chunks.
+ chunk_size(int): Size of one chunk. Default value is 200.
+
+ Returns:
+ list: Chunked items.
+ """
+ chunks = []
+
+ tupled_iterable = tuple(iterable)
+ if not tupled_iterable:
+ return chunks
+ iterable_size = len(tupled_iterable)
+ if chunk_size is None:
+ chunk_size = 200
+
+ if chunk_size < 1:
+ chunk_size = 1
+
+ for idx in range(0, iterable_size, chunk_size):
+ chunks.append(tupled_iterable[idx:idx + chunk_size])
+ return chunks
def check_regex(name, entity_type, in_schema=None, schema_patterns=None):
@@ -51,7 +176,7 @@ def check_regex(name, entity_type, in_schema=None, schema_patterns=None):
if not name_pattern:
default_pattern = "^[a-zA-Z0-9_.]*$"
- schema_obj = avalon.schema._cache.get(schema_name + ".json")
+ schema_obj = schema._cache.get(schema_name + ".json")
if not schema_obj:
name_pattern = default_pattern
else:
@@ -162,21 +287,6 @@ def from_dict_to_set(data, is_project):
return result
-def get_avalon_project_template(project_name):
- """Get avalon template
- Args:
- project_name: (string)
- Returns:
- dictionary with templates
- """
- templates = Anatomy(project_name).templates
- return {
- "workfile": templates["avalon"]["workfile"],
- "work": templates["avalon"]["work"],
- "publish": templates["avalon"]["publish"]
- }
-
-
def get_project_apps(in_app_list):
""" Application definitions for app name.
@@ -235,33 +345,19 @@ def get_hierarchical_attributes_values(
entity_ids = [item["id"] for item in entity["link"]]
- join_ent_ids = join_query_keys(entity_ids)
- join_attribute_ids = join_query_keys(attr_key_by_id.keys())
-
- queries = []
- queries.append({
- "action": "query",
- "expression": (
- "select value, configuration_id, entity_id"
- " from CustomAttributeValue"
- " where entity_id in ({}) and configuration_id in ({})"
- ).format(join_ent_ids, join_attribute_ids)
- })
-
- if hasattr(session, "call"):
- [values] = session.call(queries)
- else:
- [values] = session._call(queries)
+ values = query_custom_attributes(
+ session, list(attr_key_by_id.keys()), entity_ids, True
+ )
hier_values = {}
for key, val in defaults.items():
hier_values[key] = val
- if not values["data"]:
+ if not values:
return hier_values
values_by_entity_id = collections.defaultdict(dict)
- for item in values["data"]:
+ for item in values:
value = item["value"]
if value is None:
continue
@@ -304,7 +400,7 @@ class SyncEntitiesFactory:
" from Project where full_name is \"{}\""
)
entities_query = (
- "select id, name, type_id, parent_id, link"
+ "select id, name, type_id, parent_id, link, description"
" from TypedContext where project_id is \"{}\""
)
ignore_custom_attr_key = "avalon_ignore_sync"
@@ -341,6 +437,7 @@ def launch_setup(self, project_full_name):
}
self.create_list = []
+ self.project_created = False
self.unarchive_list = []
self.updates = collections.defaultdict(dict)
@@ -482,6 +579,10 @@ def launch_setup(self, project_full_name):
self.ft_project_id = ft_project_id
self.entities_dict = entities_dict
+ @property
+ def project_name(self):
+ return self.entities_dict[self.ft_project_id]["name"]
+
@property
def avalon_ents_by_id(self):
"""
@@ -566,9 +667,9 @@ def avalon_archived_ents(self):
(list) of assets
"""
if self._avalon_archived_ents is None:
- self._avalon_archived_ents = [
- ent for ent in self.dbcon.find({"type": "archived_asset"})
- ]
+ self._avalon_archived_ents = list(
+ get_archived_assets(self.project_name)
+ )
return self._avalon_archived_ents
@property
@@ -636,7 +737,7 @@ def subsets_by_parent_id(self):
"""
if self._subsets_by_parent_id is None:
self._subsets_by_parent_id = collections.defaultdict(list)
- for subset in self.dbcon.find({"type": "subset"}):
+ for subset in get_subsets(self.project_name):
self._subsets_by_parent_id[str(subset["parent"])].append(
subset
)
@@ -861,33 +962,6 @@ def filter_by_selection(self, event):
self.entities_dict[parent_id]["children"].remove(ftrack_id)
- def _query_custom_attributes(self, session, conf_ids, entity_ids):
- output = []
- # Prepare values to query
- attributes_joined = join_query_keys(conf_ids)
- attributes_len = len(conf_ids)
- chunk_size = int(5000 / attributes_len)
- for idx in range(0, len(entity_ids), chunk_size):
- entity_ids_joined = join_query_keys(
- entity_ids[idx:idx + chunk_size]
- )
-
- call_expr = [{
- "action": "query",
- "expression": (
- "select value, entity_id from ContextCustomAttributeValue "
- "where entity_id in ({}) and configuration_id in ({})"
- ).format(entity_ids_joined, attributes_joined)
- }]
- if hasattr(session, "call"):
- [result] = session.call(call_expr)
- else:
- [result] = session._call(call_expr)
-
- for item in result["data"]:
- output.append(item)
- return output
-
def set_cutom_attributes(self):
self.log.debug("* Preparing custom attributes")
# Get custom attributes and values
@@ -994,12 +1068,13 @@ def set_cutom_attributes(self):
copy.deepcopy(prepared_avalon_attr_ca_id)
)
- items = self._query_custom_attributes(
+ items = query_custom_attributes(
self.session,
list(attribute_key_by_id.keys()),
sync_ids
)
+ invalid_fps_items = []
for item in items:
entity_id = item["entity_id"]
attr_id = item["configuration_id"]
@@ -1012,8 +1087,24 @@ def set_cutom_attributes(self):
value = item["value"]
if convert_type:
value = convert_type(value)
+
+ if key in FPS_KEYS:
+ try:
+ value = convert_to_fps(value)
+ except InvalidFpsValue:
+ invalid_fps_items.append((entity_id, value))
self.entities_dict[entity_id][store_key][key] = value
+ if invalid_fps_items:
+ fps_msg = (
+ "These entities have invalid fps value in custom attributes"
+ )
+ items = []
+ for entity_id, value in invalid_fps_items:
+ ent_path = self.get_ent_path(entity_id)
+ items.append("{} - \"{}\"".format(ent_path, value))
+ self.report_items["error"][fps_msg] = items
+
# process hierarchical attributes
self.set_hierarchical_attribute(
hier_attrs, sync_ids, cust_attr_type_name_by_id
@@ -1046,8 +1137,15 @@ def set_hierarchical_attribute(
if key.startswith("avalon_"):
store_key = "avalon_attrs"
+ default_value = attr["default"]
+ if key in FPS_KEYS:
+ try:
+ default_value = convert_to_fps(default_value)
+ except InvalidFpsValue:
+ pass
+
self.entities_dict[self.ft_project_id][store_key][key] = (
- attr["default"]
+ default_value
)
# Add attribute ids to entities dictionary
@@ -1082,12 +1180,14 @@ def set_hierarchical_attribute(
for key, val in prepare_dict_avalon.items():
entity_dict["avalon_attrs"][key] = val
- items = self._query_custom_attributes(
+ items = query_custom_attributes(
self.session,
list(attribute_key_by_id.keys()),
- sync_ids
+ sync_ids,
+ True
)
+ invalid_fps_items = []
avalon_hier = []
for item in items:
value = item["value"]
@@ -1107,6 +1207,13 @@ def set_hierarchical_attribute(
entity_id = item["entity_id"]
key = attribute_key_by_id[attr_id]
+ if key in FPS_KEYS:
+ try:
+ value = convert_to_fps(value)
+ except InvalidFpsValue:
+ invalid_fps_items.append((entity_id, value))
+ continue
+
if key.startswith("avalon_"):
store_key = "avalon_attrs"
avalon_hier.append(key)
@@ -1114,6 +1221,16 @@ def set_hierarchical_attribute(
store_key = "hier_attrs"
self.entities_dict[entity_id][store_key][key] = value
+ if invalid_fps_items:
+ fps_msg = (
+ "These entities have invalid fps value in custom attributes"
+ )
+ items = []
+ for entity_id, value in invalid_fps_items:
+ ent_path = self.get_ent_path(entity_id)
+ items.append("{} - \"{}\"".format(ent_path, value))
+ self.report_items["error"][fps_msg] = items
+
# Get dictionary with not None hierarchical values to pull to childs
top_id = self.ft_project_id
project_values = {}
@@ -1190,10 +1307,8 @@ def _get_input_links(self, ftrack_ids):
ids_len = len(tupled_ids)
chunk_size = int(5000 / ids_len)
all_links = []
- for idx in range(0, ids_len, chunk_size):
- entity_ids_joined = join_query_keys(
- tupled_ids[idx:idx + chunk_size]
- )
+ for chunk in create_chunks(ftrack_ids, chunk_size):
+ entity_ids_joined = join_query_keys(chunk)
all_links.extend(self.session.query((
"select from_id, to_id from"
@@ -1231,6 +1346,8 @@ def prepare_ftrack_ent_data(self):
data[key] = val
if ftrack_id != self.ft_project_id:
+ data["description"] = entity["description"]
+
ent_path_items = [ent["name"] for ent in entity["link"]]
parents = ent_path_items[1:len(ent_path_items) - 1:]
@@ -1311,8 +1428,8 @@ def prepare_avalon_entities(self, ft_project_name):
# Avalon entities
self.dbcon.install()
self.dbcon.Session["AVALON_PROJECT"] = ft_project_name
- avalon_project = self.dbcon.find_one({"type": "project"})
- avalon_entities = self.dbcon.find({"type": "asset"})
+ avalon_project = get_project(ft_project_name)
+ avalon_entities = get_assets(ft_project_name)
self.avalon_project = avalon_project
self.avalon_entities = avalon_entities
@@ -1804,10 +1921,10 @@ def prepare_changes(self):
configuration_id = self.entities_dict[ftrack_id][
"avalon_attrs_id"][CUST_ATTR_ID_KEY]
- _entity_key = collections.OrderedDict({
- "configuration_id": configuration_id,
- "entity_id": ftrack_id
- })
+ _entity_key = collections.OrderedDict([
+ ("configuration_id", configuration_id),
+ ("entity_id", ftrack_id)
+ ])
self.session.recorded_operations.push(
ftrack_api.operation.UpdateEntityOperation(
@@ -1940,7 +2057,7 @@ def create_avalon_entity(self, ftrack_id):
item["_id"] = new_id
item["parent"] = self.avalon_project_id
- item["schema"] = CURRENT_DOC_SCHEMAS["asset"]
+ item["schema"] = CURRENT_ASSET_DOC_SCHEMA
item["data"]["visualParent"] = avalon_parent
new_id_str = str(new_id)
@@ -2075,8 +2192,8 @@ def create_avalon_project(self):
project_item["_id"] = new_id
project_item["parent"] = None
- project_item["schema"] = CURRENT_DOC_SCHEMAS["project"]
- project_item["config"]["schema"] = CURRENT_DOC_SCHEMAS["config"]
+ project_item["schema"] = CURRENT_PROJECT_SCHEMA
+ project_item["config"]["schema"] = CURRENT_PROJECT_CONFIG_SCHEMA
self.ftrack_avalon_mapper[self.ft_project_id] = new_id
self.avalon_ftrack_mapper[new_id] = self.ft_project_id
@@ -2092,6 +2209,7 @@ def create_avalon_project(self):
self._avalon_ents_by_name[project_item["name"]] = str(new_id)
self.create_list.append(project_item)
+ self.project_created = True
# store mongo id to ftrack entity
entity = self.entities_dict[self.ft_project_id]["entity"]
@@ -2148,46 +2266,37 @@ def _bubble_changeability(self, unchangeable_ids):
self._delete_subsets_without_asset(subsets_to_remove)
def _delete_subsets_without_asset(self, not_existing_parents):
- subset_ids = []
- version_ids = []
repre_ids = []
to_delete = []
+ subset_ids = []
for parent_id in not_existing_parents:
subsets = self.subsets_by_parent_id.get(parent_id)
if not subsets:
continue
for subset in subsets:
- if subset.get("type") != "subset":
- continue
- subset_ids.append(subset["_id"])
+ if subset.get("type") == "subset":
+ subset_ids.append(subset["_id"])
- db_subsets = self.dbcon.find({
- "_id": {"$in": subset_ids},
- "type": "subset"
- })
- if not db_subsets:
- return
-
- db_versions = self.dbcon.find({
- "parent": {"$in": subset_ids},
- "type": "version"
- })
- if db_versions:
- version_ids = [ver["_id"] for ver in db_versions]
-
- db_repres = self.dbcon.find({
- "parent": {"$in": version_ids},
- "type": "representation"
- })
- if db_repres:
- repre_ids = [repre["_id"] for repre in db_repres]
+ db_versions = get_versions(
+ self.project_name,
+ subset_ids=subset_ids,
+ fields=["_id"]
+ )
+ version_ids = [ver["_id"] for ver in db_versions]
+ db_repres = get_representations(
+ self.project_name,
+ version_ids=version_ids,
+ fields=["_id"]
+ )
+ repre_ids = [repre["_id"] for repre in db_repres]
to_delete.extend(subset_ids)
to_delete.extend(version_ids)
to_delete.extend(repre_ids)
- self.dbcon.delete_many({"_id": {"$in": to_delete}})
+ if to_delete:
+ self.dbcon.delete_many({"_id": {"$in": to_delete}})
# Probably deprecated
def _check_changeability(self, parent_id=None):
@@ -2669,8 +2778,7 @@ def regex_duplicate_interface(self):
def report(self):
items = []
- project_name = self.entities_dict[self.ft_project_id]["name"]
- title = "Synchronization report ({}):".format(project_name)
+ title = "Synchronization report ({}):".format(self.project_name)
keys = ["error", "warning", "info"]
for key in keys:
diff --git a/openpype/modules/ftrack/lib/constants.py b/openpype/modules/ftrack/lib/constants.py
new file mode 100644
index 00000000000..636dcfbc3d7
--- /dev/null
+++ b/openpype/modules/ftrack/lib/constants.py
@@ -0,0 +1,20 @@
+# Group name of custom attributes
+CUST_ATTR_GROUP = "openpype"
+
+# name of Custom attribute that stores mongo_id from avalon db
+CUST_ATTR_ID_KEY = "avalon_mongo_id"
+# Auto sync of project
+CUST_ATTR_AUTO_SYNC = "avalon_auto_sync"
+
+# Applications custom attribute name
+CUST_ATTR_APPLICATIONS = "applications"
+# Environment tools custom attribute
+CUST_ATTR_TOOLS = "tools_env"
+# Intent custom attribute name
+CUST_ATTR_INTENT = "intent"
+
+FPS_KEYS = {
+ "fps",
+ # For development purposes
+ "fps_string"
+}
diff --git a/openpype/modules/ftrack/lib/credentials.py b/openpype/modules/ftrack/lib/credentials.py
new file mode 100644
index 00000000000..2eb64254d1b
--- /dev/null
+++ b/openpype/modules/ftrack/lib/credentials.py
@@ -0,0 +1,109 @@
+import os
+import ftrack_api
+
+try:
+ from urllib.parse import urlparse
+except ImportError:
+ from urlparse import urlparse
+
+
+from openpype.lib import OpenPypeSecureRegistry
+
+USERNAME_KEY = "username"
+API_KEY_KEY = "api_key"
+
+
+def get_ftrack_hostname(ftrack_server=None):
+ if not ftrack_server:
+ ftrack_server = os.environ.get("FTRACK_SERVER")
+
+ if not ftrack_server:
+ return None
+
+ if "//" not in ftrack_server:
+ ftrack_server = "//" + ftrack_server
+
+ return urlparse(ftrack_server).hostname
+
+
+def _get_ftrack_secure_key(hostname, key):
+ """Secure item key for entered hostname."""
+ return "/".join(("ftrack", hostname, key))
+
+
+def get_credentials(ftrack_server=None):
+ output = {
+ USERNAME_KEY: None,
+ API_KEY_KEY: None
+ }
+ hostname = get_ftrack_hostname(ftrack_server)
+ if not hostname:
+ return output
+
+ username_name = _get_ftrack_secure_key(hostname, USERNAME_KEY)
+ api_key_name = _get_ftrack_secure_key(hostname, API_KEY_KEY)
+
+ username_registry = OpenPypeSecureRegistry(username_name)
+ api_key_registry = OpenPypeSecureRegistry(api_key_name)
+
+ output[USERNAME_KEY] = username_registry.get_item(USERNAME_KEY, None)
+ output[API_KEY_KEY] = api_key_registry.get_item(API_KEY_KEY, None)
+
+ return output
+
+
+def save_credentials(username, api_key, ftrack_server=None):
+ hostname = get_ftrack_hostname(ftrack_server)
+ username_name = _get_ftrack_secure_key(hostname, USERNAME_KEY)
+ api_key_name = _get_ftrack_secure_key(hostname, API_KEY_KEY)
+
+ # Clear credentials
+ clear_credentials(ftrack_server)
+
+ username_registry = OpenPypeSecureRegistry(username_name)
+ api_key_registry = OpenPypeSecureRegistry(api_key_name)
+
+ username_registry.set_item(USERNAME_KEY, username)
+ api_key_registry.set_item(API_KEY_KEY, api_key)
+
+
+def clear_credentials(ftrack_server=None):
+ hostname = get_ftrack_hostname(ftrack_server)
+ username_name = _get_ftrack_secure_key(hostname, USERNAME_KEY)
+ api_key_name = _get_ftrack_secure_key(hostname, API_KEY_KEY)
+
+ username_registry = OpenPypeSecureRegistry(username_name)
+ api_key_registry = OpenPypeSecureRegistry(api_key_name)
+
+ current_username = username_registry.get_item(USERNAME_KEY, None)
+ current_api_key = api_key_registry.get_item(API_KEY_KEY, None)
+
+ if current_username is not None:
+ username_registry.delete_item(USERNAME_KEY)
+
+ if current_api_key is not None:
+ api_key_registry.delete_item(API_KEY_KEY)
+
+
+def check_credentials(username, api_key, ftrack_server=None):
+ if not ftrack_server:
+ ftrack_server = os.environ.get("FTRACK_SERVER")
+
+ if not ftrack_server or not username or not api_key:
+ return False
+
+ user_exists = False
+ try:
+ session = ftrack_api.Session(
+ server_url=ftrack_server,
+ api_key=api_key,
+ api_user=username
+ )
+ # Validated that the username actually exists
+ user = session.query("User where username is \"{}\"".format(username))
+ user_exists = user is not None
+ session.close()
+
+ except Exception:
+ pass
+ return user_exists
diff --git a/openpype/modules/default_modules/ftrack/lib/custom_attributes.json b/openpype/modules/ftrack/lib/custom_attributes.json
similarity index 100%
rename from openpype/modules/default_modules/ftrack/lib/custom_attributes.json
rename to openpype/modules/ftrack/lib/custom_attributes.json
diff --git a/openpype/modules/default_modules/ftrack/lib/custom_attributes.py b/openpype/modules/ftrack/lib/custom_attributes.py
similarity index 77%
rename from openpype/modules/default_modules/ftrack/lib/custom_attributes.py
rename to openpype/modules/ftrack/lib/custom_attributes.py
index 53facd4ab22..2f538153680 100644
--- a/openpype/modules/default_modules/ftrack/lib/custom_attributes.py
+++ b/openpype/modules/ftrack/lib/custom_attributes.py
@@ -17,7 +17,7 @@ def default_custom_attributes_definition():
def app_definitions_from_app_manager(app_manager):
_app_definitions = []
for app_name, app in app_manager.applications.items():
- if app.enabled and app.is_host:
+ if app.enabled:
_app_definitions.append(
(app_name, app.full_label)
)
@@ -88,26 +88,36 @@ def join_query_keys(keys):
return ",".join(["\"{}\"".format(key) for key in keys])
-def query_custom_attributes(session, conf_ids, entity_ids, table_name=None):
+def query_custom_attributes(
+ session, conf_ids, entity_ids, only_set_values=False
+):
"""Query custom attribute values from ftrack database.
Using ftrack call method result may differ based on used table name and
version of ftrack server.
+ For hierarchical attributes you shou always use `only_set_values=True`
+ otherwise result will be default value of custom attribute and it would not
+ be possible to differentiate if value is set on entity or default value is
+ used.
+
Args:
session(ftrack_api.Session): Connected ftrack session.
conf_id(list, set, tuple): Configuration(attribute) ids which are
queried.
entity_ids(list, set, tuple): Entity ids for which are values queried.
- table_name(str): Table nam from which values are queried. Not
- recommended to change until you know what it means.
+ only_set_values(bool): Entities that don't have explicitly set
+ value won't return a value. If is set to False then default custom
+ attribute value is returned if value is not set.
"""
output = []
# Just skip
if not conf_ids or not entity_ids:
return output
- if table_name is None:
+ if only_set_values:
+ table_name = "CustomAttributeValue"
+ else:
table_name = "ContextCustomAttributeValue"
# Prepare values to query
@@ -122,19 +132,16 @@ def query_custom_attributes(session, conf_ids, entity_ids, table_name=None):
entity_ids_joined = join_query_keys(
entity_ids[idx:idx + chunk_size]
)
-
- call_expr = [{
- "action": "query",
- "expression": (
- "select value, entity_id from {}"
- " where entity_id in ({}) and configuration_id in ({})"
- ).format(table_name, entity_ids_joined, attributes_joined)
- }]
- if hasattr(session, "call"):
- [result] = session.call(call_expr)
- else:
- [result] = session._call(call_expr)
-
- for item in result["data"]:
- output.append(item)
+ output.extend(
+ session.query(
+ (
+ "select value, entity_id, configuration_id from {}"
+ " where entity_id in ({}) and configuration_id in ({})"
+ ).format(
+ table_name,
+ entity_ids_joined,
+ attributes_joined
+ )
+ ).all()
+ )
return output
diff --git a/openpype/modules/default_modules/ftrack/lib/ftrack_action_handler.py b/openpype/modules/ftrack/lib/ftrack_action_handler.py
similarity index 100%
rename from openpype/modules/default_modules/ftrack/lib/ftrack_action_handler.py
rename to openpype/modules/ftrack/lib/ftrack_action_handler.py
diff --git a/openpype/modules/default_modules/ftrack/lib/ftrack_base_handler.py b/openpype/modules/ftrack/lib/ftrack_base_handler.py
similarity index 99%
rename from openpype/modules/default_modules/ftrack/lib/ftrack_base_handler.py
rename to openpype/modules/ftrack/lib/ftrack_base_handler.py
index 2130abc20c1..c0b03f8a417 100644
--- a/openpype/modules/default_modules/ftrack/lib/ftrack_base_handler.py
+++ b/openpype/modules/ftrack/lib/ftrack_base_handler.py
@@ -6,7 +6,7 @@
import datetime
import traceback
import time
-from openpype.api import Logger
+from openpype.lib import Logger
from openpype.settings import get_project_settings
import ftrack_api
@@ -52,7 +52,7 @@ def join_query_keys(keys):
def __init__(self, session):
'''Expects a ftrack_api.Session instance'''
- self.log = Logger().get_logger(self.__class__.__name__)
+ self.log = Logger.get_logger(self.__class__.__name__)
if not(
isinstance(session, ftrack_api.session.Session) or
isinstance(session, ftrack_server.lib.SocketSession)
@@ -535,7 +535,7 @@ def trigger_action(
)
def trigger_event(
- self, topic, event_data={}, session=None, source=None,
+ self, topic, event_data=None, session=None, source=None,
event=None, on_error="ignore"
):
if session is None:
@@ -543,6 +543,9 @@ def trigger_event(
if not source and event:
source = event.get("source")
+
+ if event_data is None:
+ event_data = {}
# Create and trigger event
event = ftrack_api.event.base.Event(
topic=topic,
diff --git a/openpype/modules/default_modules/ftrack/lib/ftrack_event_handler.py b/openpype/modules/ftrack/lib/ftrack_event_handler.py
similarity index 97%
rename from openpype/modules/default_modules/ftrack/lib/ftrack_event_handler.py
rename to openpype/modules/ftrack/lib/ftrack_event_handler.py
index af565c54213..0a70b0e3016 100644
--- a/openpype/modules/default_modules/ftrack/lib/ftrack_event_handler.py
+++ b/openpype/modules/ftrack/lib/ftrack_event_handler.py
@@ -44,7 +44,7 @@ def _translate_event(self, event, session=None):
return self._get_entities(
event,
session,
- ignore=['socialfeed', 'socialnotification']
+ ignore=['socialfeed', 'socialnotification', 'team']
)
def get_project_name_from_event(self, session, event, project_id):
diff --git a/openpype/modules/default_modules/ftrack/lib/settings.py b/openpype/modules/ftrack/lib/settings.py
similarity index 100%
rename from openpype/modules/default_modules/ftrack/lib/settings.py
rename to openpype/modules/ftrack/lib/settings.py
diff --git a/openpype/modules/default_modules/ftrack/plugins/_unused_publish/integrate_ftrack_comments.py b/openpype/modules/ftrack/plugins/_unused_publish/integrate_ftrack_comments.py
similarity index 100%
rename from openpype/modules/default_modules/ftrack/plugins/_unused_publish/integrate_ftrack_comments.py
rename to openpype/modules/ftrack/plugins/_unused_publish/integrate_ftrack_comments.py
diff --git a/openpype/modules/ftrack/plugins/publish/collect_custom_attributes_data.py b/openpype/modules/ftrack/plugins/publish/collect_custom_attributes_data.py
new file mode 100644
index 00000000000..43fa3bc3f8d
--- /dev/null
+++ b/openpype/modules/ftrack/plugins/publish/collect_custom_attributes_data.py
@@ -0,0 +1,148 @@
+"""
+Requires:
+ context > ftrackSession
+ context > ftrackEntity
+ instance > ftrackEntity
+
+Provides:
+ instance > customData > ftrack
+"""
+import copy
+
+import pyblish.api
+
+
+class CollectFtrackCustomAttributeData(pyblish.api.ContextPlugin):
+ """Collect custom attribute values and store them to customData.
+
+ Data are stored into each instance in context under
+ instance.data["customData"]["ftrack"].
+
+ Hierarchical attributes are not looked up properly for that functionality
+ custom attribute values lookup must be extended.
+ """
+
+ order = pyblish.api.CollectorOrder + 0.4992
+ label = "Collect Ftrack Custom Attribute Data"
+
+ # Name of custom attributes for which will be look for
+ custom_attribute_keys = []
+
+ def process(self, context):
+ if not self.custom_attribute_keys:
+ self.log.info("Custom attribute keys are not set. Skipping")
+ return
+
+ ftrack_entities_by_id = {}
+ default_entity_id = None
+
+ context_entity = context.data.get("ftrackEntity")
+ if context_entity:
+ entity_id = context_entity["id"]
+ default_entity_id = entity_id
+ ftrack_entities_by_id[entity_id] = context_entity
+
+ instances_by_entity_id = {
+ default_entity_id: []
+ }
+ for instance in context:
+ entity = instance.data.get("ftrackEntity")
+ if not entity:
+ instances_by_entity_id[default_entity_id].append(instance)
+ continue
+
+ entity_id = entity["id"]
+ ftrack_entities_by_id[entity_id] = entity
+ if entity_id not in instances_by_entity_id:
+ instances_by_entity_id[entity_id] = []
+ instances_by_entity_id[entity_id].append(instance)
+
+ if not ftrack_entities_by_id:
+ self.log.info("Ftrack entities are not set. Skipping")
+ return
+
+ session = context.data["ftrackSession"]
+ custom_attr_key_by_id = self.query_attr_confs(session)
+ if not custom_attr_key_by_id:
+ self.log.info((
+ "Didn't find any of defined custom attributes {}"
+ ).format(", ".join(self.custom_attribute_keys)))
+ return
+
+ entity_ids = list(instances_by_entity_id.keys())
+ values_by_entity_id = self.query_attr_values(
+ session, entity_ids, custom_attr_key_by_id
+ )
+
+ for entity_id, instances in instances_by_entity_id.items():
+ if entity_id not in values_by_entity_id:
+ # Use defaut empty values
+ entity_id = None
+
+ for instance in instances:
+ value = copy.deepcopy(values_by_entity_id[entity_id])
+ if "customData" not in instance.data:
+ instance.data["customData"] = {}
+ instance.data["customData"]["ftrack"] = value
+ instance_label = (
+ instance.data.get("label") or instance.data["name"]
+ )
+ self.log.debug((
+ "Added ftrack custom data to instance \"{}\": {}"
+ ).format(instance_label, value))
+
+ def query_attr_values(self, session, entity_ids, custom_attr_key_by_id):
+ # Prepare values for query
+ entity_ids_joined = ",".join([
+ '"{}"'.format(entity_id)
+ for entity_id in entity_ids
+ ])
+ conf_ids_joined = ",".join([
+ '"{}"'.format(conf_id)
+ for conf_id in custom_attr_key_by_id.keys()
+ ])
+ # Query custom attribute values
+ value_items = session.query(
+ (
+ "select value, entity_id, configuration_id"
+ " from CustomAttributeValue"
+ " where entity_id in ({}) and configuration_id in ({})"
+ ).format(
+ entity_ids_joined,
+ conf_ids_joined
+ )
+ ).all()
+
+ # Prepare default value output per entity id
+ values_by_key = {
+ key: None for key in self.custom_attribute_keys
+ }
+ # Prepare all entity ids that were queried
+ values_by_entity_id = {
+ entity_id: copy.deepcopy(values_by_key)
+ for entity_id in entity_ids
+ }
+ # Add none entity id which is used as default value
+ values_by_entity_id[None] = copy.deepcopy(values_by_key)
+ # Go through queried data and store them
+ for item in value_items:
+ conf_id = item["configuration_id"]
+ conf_key = custom_attr_key_by_id[conf_id]
+ entity_id = item["entity_id"]
+ values_by_entity_id[entity_id][conf_key] = item["value"]
+ return values_by_entity_id
+
+ def query_attr_confs(self, session):
+ custom_attributes = set(self.custom_attribute_keys)
+ cust_attrs_query = (
+ "select id, key from CustomAttributeConfiguration"
+ " where key in ({})"
+ ).format(", ".join(
+ ["\"{}\"".format(attr_name) for attr_name in custom_attributes]
+ ))
+
+ custom_attr_confs = session.query(cust_attrs_query).all()
+ return {
+ conf["id"]: conf["key"]
+ for conf in custom_attr_confs
+ }
diff --git a/openpype/modules/ftrack/plugins/publish/collect_ftrack_api.py b/openpype/modules/ftrack/plugins/publish/collect_ftrack_api.py
new file mode 100644
index 00000000000..e13b7e65cdf
--- /dev/null
+++ b/openpype/modules/ftrack/plugins/publish/collect_ftrack_api.py
@@ -0,0 +1,231 @@
+import logging
+import pyblish.api
+
+from openpype.pipeline import legacy_io
+
+
+class CollectFtrackApi(pyblish.api.ContextPlugin):
+ """ Collects an ftrack session and the current task id. """
+
+ order = pyblish.api.CollectorOrder + 0.4991
+ label = "Collect Ftrack Api"
+
+ def process(self, context):
+ ftrack_log = logging.getLogger('ftrack_api')
+ ftrack_log.setLevel(logging.WARNING)
+ ftrack_log = logging.getLogger('ftrack_api_old')
+ ftrack_log.setLevel(logging.WARNING)
+
+ # Collect session
+ # NOTE Import python module here to know if import was successful
+ import ftrack_api
+
+ session = ftrack_api.Session(auto_connect_event_hub=False)
+ self.log.debug("Ftrack user: \"{0}\"".format(session.api_user))
+
+ # Collect task
+ project_name = legacy_io.Session["AVALON_PROJECT"]
+ asset_name = legacy_io.Session["AVALON_ASSET"]
+ task_name = legacy_io.Session["AVALON_TASK"]
+
+ # Find project entity
+ project_query = 'Project where full_name is "{0}"'.format(project_name)
+ self.log.debug("Project query: < {0} >".format(project_query))
+ project_entities = list(session.query(project_query).all())
+ if len(project_entities) == 0:
+ raise AssertionError(
+ "Project \"{0}\" not found in Ftrack.".format(project_name)
+ )
+ # QUESTION Is possible to happen?
+ elif len(project_entities) > 1:
+ raise AssertionError((
+ "Found more than one project with name \"{0}\" in Ftrack."
+ ).format(project_name))
+
+ project_entity = project_entities[0]
+
+ self.log.debug("Project found: {0}".format(project_entity))
+
+ asset_entity = None
+ if asset_name:
+ # Find asset entity
+ entity_query = (
+ 'TypedContext where project_id is "{0}"'
+ ' and name is "{1}"'
+ ).format(project_entity["id"], asset_name)
+ self.log.debug("Asset entity query: < {0} >".format(entity_query))
+ asset_entities = []
+ for entity in session.query(entity_query).all():
+ # Skip tasks
+ if entity.entity_type.lower() != "task":
+ asset_entities.append(entity)
+
+ if len(asset_entities) == 0:
+ raise AssertionError((
+ "Entity with name \"{0}\" not found"
+ " in Ftrack project \"{1}\"."
+ ).format(asset_name, project_name))
+
+ elif len(asset_entities) > 1:
+ raise AssertionError((
+ "Found more than one entity with name \"{0}\""
+ " in Ftrack project \"{1}\"."
+ ).format(asset_name, project_name))
+
+ asset_entity = asset_entities[0]
+
+ self.log.debug("Asset found: {0}".format(asset_entity))
+
+ task_entity = None
+ # Find task entity if task is set
+ if not asset_entity:
+ self.log.warning(
+ "Asset entity is not set. Skipping query of task entity."
+ )
+ elif not task_name:
+ self.log.warning("Task name is not set.")
+ else:
+ task_query = (
+ 'Task where name is "{0}" and parent_id is "{1}"'
+ ).format(task_name, asset_entity["id"])
+ self.log.debug("Task entity query: < {0} >".format(task_query))
+ task_entity = session.query(task_query).first()
+ if not task_entity:
+ self.log.warning(
+ "Task entity with name \"{0}\" was not found.".format(
+ task_name
+ )
+ )
+ else:
+ self.log.debug("Task entity found: {0}".format(task_entity))
+
+ context.data["ftrackSession"] = session
+ context.data["ftrackPythonModule"] = ftrack_api
+ context.data["ftrackProject"] = project_entity
+ context.data["ftrackEntity"] = asset_entity
+ context.data["ftrackTask"] = task_entity
+
+ self.per_instance_process(context, asset_entity, task_entity)
+
+ def per_instance_process(
+ self, context, context_asset_entity, context_task_entity
+ ):
+ context_task_name = None
+ context_asset_name = None
+ if context_asset_entity:
+ context_asset_name = context_asset_entity["name"]
+ if context_task_entity:
+ context_task_name = context_task_entity["name"]
+ instance_by_asset_and_task = {}
+ for instance in context:
+ self.log.debug(
+ "Checking entities of instance \"{}\"".format(str(instance))
+ )
+ instance_asset_name = instance.data.get("asset")
+ instance_task_name = instance.data.get("task")
+
+ if not instance_asset_name and not instance_task_name:
+ self.log.debug("Instance does not have set context keys.")
+ instance.data["ftrackEntity"] = context_asset_entity
+ instance.data["ftrackTask"] = context_task_entity
+ continue
+
+ elif instance_asset_name and instance_task_name:
+ if (
+ instance_asset_name == context_asset_name
+ and instance_task_name == context_task_name
+ ):
+ self.log.debug((
+ "Instance's context is same as in publish context."
+ " Asset: {} | Task: {}"
+ ).format(context_asset_name, context_task_name))
+ instance.data["ftrackEntity"] = context_asset_entity
+ instance.data["ftrackTask"] = context_task_entity
+ continue
+ asset_name = instance_asset_name
+ task_name = instance_task_name
+
+ elif instance_task_name:
+ if instance_task_name == context_task_name:
+ self.log.debug((
+ "Instance's context task is same as in publish"
+ " context. Task: {}"
+ ).format(context_task_name))
+ instance.data["ftrackEntity"] = context_asset_entity
+ instance.data["ftrackTask"] = context_task_entity
+ continue
+
+ asset_name = context_asset_name
+ task_name = instance_task_name
+
+ elif instance_asset_name:
+ if instance_asset_name == context_asset_name:
+ self.log.debug((
+ "Instance's context asset is same as in publish"
+ " context. Asset: {}"
+ ).format(context_asset_name))
+ instance.data["ftrackEntity"] = context_asset_entity
+ instance.data["ftrackTask"] = context_task_entity
+ continue
+
+ # Do not use context's task name
+ task_name = instance_task_name
+ asset_name = instance_asset_name
+
+ if asset_name not in instance_by_asset_and_task:
+ instance_by_asset_and_task[asset_name] = {}
+
+ if task_name not in instance_by_asset_and_task[asset_name]:
+ instance_by_asset_and_task[asset_name][task_name] = []
+ instance_by_asset_and_task[asset_name][task_name].append(instance)
+
+ if not instance_by_asset_and_task:
+ return
+
+ session = context.data["ftrackSession"]
+ project_entity = context.data["ftrackProject"]
+ asset_names = set()
+ for asset_name in instance_by_asset_and_task.keys():
+ asset_names.add(asset_name)
+
+ joined_asset_names = ",".join([
+ "\"{}\"".format(name)
+ for name in asset_names
+ ])
+ entities = session.query((
+ "TypedContext where project_id is \"{}\" and name in ({})"
+ ).format(project_entity["id"], joined_asset_names)).all()
+
+ entities_by_name = {
+ entity["name"]: entity
+ for entity in entities
+ }
+
+ for asset_name, by_task_data in instance_by_asset_and_task.items():
+ entity = entities_by_name.get(asset_name)
+ task_entity_by_name = {}
+ if not entity:
+ self.log.warning((
+ "Didn't find entity with name \"{}\" in Project \"{}\""
+ ).format(asset_name, project_entity["full_name"]))
+ else:
+ task_entities = session.query((
+ "select id, name from Task where parent_id is \"{}\""
+ ).format(entity["id"])).all()
+ for task_entity in task_entities:
+ task_name_low = task_entity["name"].lower()
+ task_entity_by_name[task_name_low] = task_entity
+
+ for task_name, instances in by_task_data.items():
+ task_entity = None
+ if task_name and entity:
+ task_entity = task_entity_by_name.get(task_name.lower())
+
+ for instance in instances:
+ instance.data["ftrackEntity"] = entity
+ instance.data["ftrackTask"] = task_entity
+
+ self.log.debug((
+ "Instance {} has own ftrack entities"
+ " as has different context. TypedContext: {} Task: {}"
+ ).format(str(instance), str(entity), str(task_entity)))
diff --git a/openpype/modules/default_modules/ftrack/plugins/publish/collect_ftrack_family.py b/openpype/modules/ftrack/plugins/publish/collect_ftrack_family.py
similarity index 84%
rename from openpype/modules/default_modules/ftrack/plugins/publish/collect_ftrack_family.py
rename to openpype/modules/ftrack/plugins/publish/collect_ftrack_family.py
index 70030acad92..576a7d36c4c 100644
--- a/openpype/modules/default_modules/ftrack/plugins/publish/collect_ftrack_family.py
+++ b/openpype/modules/ftrack/plugins/publish/collect_ftrack_family.py
@@ -6,9 +6,9 @@
instance -> families ([])
"""
import pyblish.api
-import avalon.api
-from openpype.lib.plugin_tools import filter_profiles
+from openpype.pipeline import legacy_io
+from openpype.lib import filter_profiles
class CollectFtrackFamily(pyblish.api.InstancePlugin):
@@ -25,7 +25,7 @@ class CollectFtrackFamily(pyblish.api.InstancePlugin):
based on 'families' (editorial drives it by presence of 'review')
"""
label = "Collect Ftrack Family"
- order = pyblish.api.CollectorOrder + 0.4998
+ order = pyblish.api.CollectorOrder + 0.4990
profiles = None
@@ -34,9 +34,10 @@ def process(self, instance):
self.log.warning("No profiles present for adding Ftrack family")
return
+ add_ftrack_family = False
task_name = instance.data.get("task",
- avalon.api.Session["AVALON_TASK"])
- host_name = avalon.api.Session["AVALON_APP"]
+ legacy_io.Session["AVALON_TASK"])
+ host_name = legacy_io.Session["AVALON_APP"]
family = instance.data["family"]
filtering_criteria = {
@@ -53,6 +54,8 @@ def process(self, instance):
additional_filters = profile.get("advanced_filtering")
if additional_filters:
+ self.log.info("'{}' families used for additional filtering".
+ format(families))
add_ftrack_family = self._get_add_ftrack_f_from_addit_filters(
additional_filters,
families,
@@ -69,6 +72,13 @@ def process(self, instance):
else:
instance.data["families"] = ["ftrack"]
+ result_str = "Adding"
+ if not add_ftrack_family:
+ result_str = "Not adding"
+ self.log.info("{} 'ftrack' family for instance with '{}'".format(
+ result_str, family
+ ))
+
def _get_add_ftrack_f_from_addit_filters(self,
additional_filters,
families,
diff --git a/openpype/modules/default_modules/ftrack/plugins/publish/collect_local_ftrack_creds.py b/openpype/modules/ftrack/plugins/publish/collect_local_ftrack_creds.py
similarity index 100%
rename from openpype/modules/default_modules/ftrack/plugins/publish/collect_local_ftrack_creds.py
rename to openpype/modules/ftrack/plugins/publish/collect_local_ftrack_creds.py
diff --git a/openpype/modules/ftrack/plugins/publish/collect_username.py b/openpype/modules/ftrack/plugins/publish/collect_username.py
new file mode 100644
index 00000000000..798f3960a8e
--- /dev/null
+++ b/openpype/modules/ftrack/plugins/publish/collect_username.py
@@ -0,0 +1,73 @@
+"""Loads publishing context from json and continues in publish process.
+
+Should run before 'CollectAnatomyContextData' so the user on context is
+changed before it's stored to context anatomy data or instance anatomy data.
+
+Requires:
+ anatomy -> context["anatomy"] *(pyblish.api.CollectorOrder - 0.11)
+
+Provides:
+ context, instances -> All data from previous publishing process.
+"""
+
+import ftrack_api
+import os
+
+import pyblish.api
+
+
+class CollectUsernameForWebpublish(pyblish.api.ContextPlugin):
+ """
+ Translates user email to Ftrack username.
+
+ Emails in Ftrack are same as company's Slack, username is needed to
+ load data to Ftrack.
+
+ Expects "pype.club" user created on Ftrack and FTRACK_BOT_API_KEY env
+ var set up.
+
+ Resets `context.data["user"] to correctly populate `version.author` and
+ `representation.context.username`
+
+ """
+ order = pyblish.api.CollectorOrder + 0.0015
+ label = "Collect ftrack username"
+ hosts = ["webpublisher", "photoshop"]
+ targets = ["remotepublish", "filespublish", "tvpaint_worker"]
+
+ def process(self, context):
+ self.log.info("{}".format(self.__class__.__name__))
+ os.environ["FTRACK_API_USER"] = os.environ["FTRACK_BOT_API_USER"]
+ os.environ["FTRACK_API_KEY"] = os.environ["FTRACK_BOT_API_KEY"]
+
+ # for publishes with studio processing
+ user_email = os.environ.get("USER_EMAIL")
+ self.log.debug("Email from env:: {}".format(user_email))
+ if not user_email:
+ # for basic webpublishes
+ for instance in context:
+ user_email = instance.data.get("user_email")
+ self.log.debug("Email from instance:: {}".format(user_email))
+ break
+
+ if not user_email:
+ self.log.info("No email found")
+ return
+
+ session = ftrack_api.Session(auto_connect_event_hub=False)
+ user = session.query(
+ "User where email like '{}'".format(user_email)
+ ).first()
+
+ if not user:
+ raise ValueError(
+ "Couldn't find user with {} email".format(user_email))
+
+ username = user.get("username")
+ self.log.debug("Resolved ftrack username:: {}".format(username))
+ os.environ["FTRACK_API_USER"] = username
+
+ burnin_name = username
+ if '@' in burnin_name:
+ burnin_name = burnin_name[:burnin_name.index('@')]
+ context.data["user"] = burnin_name
diff --git a/openpype/modules/ftrack/plugins/publish/integrate_ftrack_api.py b/openpype/modules/ftrack/plugins/publish/integrate_ftrack_api.py
new file mode 100644
index 00000000000..159e60024d7
--- /dev/null
+++ b/openpype/modules/ftrack/plugins/publish/integrate_ftrack_api.py
@@ -0,0 +1,663 @@
+"""Integrate components into ftrack
+
+Requires:
+ context -> ftrackSession - connected ftrack.Session
+ instance -> ftrackComponentsList - list of components to integrate
+
+Provides:
+ instance -> ftrackIntegratedAssetVersionsData
+ # legacy
+ instance -> ftrackIntegratedAssetVersions
+"""
+
+import os
+import sys
+import collections
+
+import six
+import pyblish.api
+import clique
+
+
+class IntegrateFtrackApi(pyblish.api.InstancePlugin):
+ """ Commit components to server. """
+
+ order = pyblish.api.IntegratorOrder + 0.499
+ label = "Integrate Ftrack Api"
+ families = ["ftrack"]
+
+ def process(self, instance):
+ component_list = instance.data.get("ftrackComponentsList")
+ if not component_list:
+ self.log.info(
+ "Instance don't have components to integrate to Ftrack."
+ " Skipping."
+ )
+ return
+
+ context = instance.context
+ session = context.data["ftrackSession"]
+
+ parent_entity = None
+ default_asset_name = None
+ # If instance has set "ftrackEntity" or "ftrackTask" then use them from
+ # instance. Even if they are set to None. If they are set to None it
+ # has a reason. (like has different context)
+ if "ftrackEntity" in instance.data or "ftrackTask" in instance.data:
+ task_entity = instance.data.get("ftrackTask")
+ parent_entity = instance.data.get("ftrackEntity")
+
+ elif "ftrackEntity" in context.data or "ftrackTask" in context.data:
+ task_entity = context.data.get("ftrackTask")
+ parent_entity = context.data.get("ftrackEntity")
+
+ if task_entity:
+ default_asset_name = task_entity["name"]
+ parent_entity = task_entity["parent"]
+
+ if parent_entity is None:
+ self.log.info((
+ "Skipping ftrack integration. Instance \"{}\" does not"
+ " have specified ftrack entities."
+ ).format(str(instance)))
+ return
+
+ if not default_asset_name:
+ default_asset_name = parent_entity["name"]
+
+ # Change status on task
+ asset_version_status_ids_by_name = {}
+ project_entity = instance.context.data.get("ftrackProject")
+ if project_entity:
+ project_schema = project_entity["project_schema"]
+ asset_version_statuses = (
+ project_schema.get_statuses("AssetVersion")
+ )
+ asset_version_status_ids_by_name = {
+ status["name"].lower(): status["id"]
+ for status in asset_version_statuses
+ }
+
+ self._set_task_status(instance, project_entity, task_entity, session)
+
+ # Prepare AssetTypes
+ asset_types_by_short = self._ensure_asset_types_exists(
+ session, component_list
+ )
+ self._fill_component_locations(session, component_list)
+
+ asset_versions_data_by_id = {}
+ used_asset_versions = []
+
+ # Iterate over components and publish
+ for data in component_list:
+ self.log.debug("data: {}".format(data))
+
+ # AssetType
+ asset_type_short = data["assettype_data"]["short"]
+ asset_type_entity = asset_types_by_short[asset_type_short]
+
+ # Asset
+ asset_data = data.get("asset_data") or {}
+ if "name" not in asset_data:
+ asset_data["name"] = default_asset_name
+ asset_entity = self._ensure_asset_exists(
+ session,
+ asset_data,
+ asset_type_entity["id"],
+ parent_entity["id"]
+ )
+
+ # Asset Version
+ asset_version_data = data.get("assetversion_data") or {}
+ asset_version_entity = self._ensure_asset_version_exists(
+ session,
+ asset_version_data,
+ asset_entity["id"],
+ task_entity,
+ asset_version_status_ids_by_name
+ )
+
+ # Store asset version and components items that were
+ version_id = asset_version_entity["id"]
+ if version_id not in asset_versions_data_by_id:
+ asset_versions_data_by_id[version_id] = {
+ "asset_version": asset_version_entity,
+ "component_items": []
+ }
+
+ asset_versions_data_by_id[version_id]["component_items"].append(
+ data
+ )
+
+ # Backwards compatibility
+ if asset_version_entity not in used_asset_versions:
+ used_asset_versions.append(asset_version_entity)
+
+ self._create_components(session, asset_versions_data_by_id)
+
+ instance.data["ftrackIntegratedAssetVersionsData"] = (
+ asset_versions_data_by_id
+ )
+
+ # Backwards compatibility
+ asset_versions_key = "ftrackIntegratedAssetVersions"
+ if asset_versions_key not in instance.data:
+ instance.data[asset_versions_key] = []
+
+ for asset_version in used_asset_versions:
+ if asset_version not in instance.data[asset_versions_key]:
+ instance.data[asset_versions_key].append(asset_version)
+
+ def _set_task_status(self, instance, project_entity, task_entity, session):
+ if not project_entity:
+ self.log.info("Task status won't be set, project is not known.")
+ return
+
+ if not task_entity:
+ self.log.info("Task status won't be set, task is not known.")
+ return
+
+ status_name = instance.context.data.get("ftrackStatus")
+ if not status_name:
+ self.log.info("Ftrack status name is not set.")
+ return
+
+ self.log.debug(
+ "Ftrack status name will be (maybe) set to \"{}\"".format(
+ status_name
+ )
+ )
+
+ project_schema = project_entity["project_schema"]
+ task_statuses = project_schema.get_statuses(
+ "Task", task_entity["type_id"]
+ )
+ task_statuses_by_low_name = {
+ status["name"].lower(): status for status in task_statuses
+ }
+ status = task_statuses_by_low_name.get(status_name.lower())
+ if not status:
+ self.log.warning((
+ "Task status \"{}\" won't be set,"
+ " status is now allowed on task type \"{}\"."
+ ).format(status_name, task_entity["type"]["name"]))
+ return
+
+ self.log.info("Setting task status to \"{}\"".format(status_name))
+ task_entity["status"] = status
+ try:
+ session.commit()
+ except Exception:
+ tp, value, tb = sys.exc_info()
+ session.rollback()
+ session._configure_locations()
+ six.reraise(tp, value, tb)
+
+ def _fill_component_locations(self, session, component_list):
+ components_by_location_name = collections.defaultdict(list)
+ components_by_location_id = collections.defaultdict(list)
+ for component_item in component_list:
+ # Location entity can be prefilled
+ # - this is not recommended as connection to ftrack server may
+ # be lost and in that case the entity is not valid when gets
+ # to this plugin
+ location = component_item.get("component_location")
+ if location is not None:
+ continue
+
+ # Collect location id
+ location_id = component_item.get("component_location_id")
+ if location_id:
+ components_by_location_id[location_id].append(
+ component_item
+ )
+ continue
+
+ location_name = component_item.get("component_location_name")
+ if location_name:
+ components_by_location_name[location_name].append(
+ component_item
+ )
+ continue
+
+ # Skip if there is nothing to do
+ if not components_by_location_name and not components_by_location_id:
+ return
+
+ # Query locations
+ query_filters = []
+ if components_by_location_id:
+ joined_location_ids = ",".join([
+ '"{}"'.format(location_id)
+ for location_id in components_by_location_id
+ ])
+ query_filters.append("id in ({})".format(joined_location_ids))
+
+ if components_by_location_name:
+ joined_location_names = ",".join([
+ '"{}"'.format(location_name)
+ for location_name in components_by_location_name
+ ])
+ query_filters.append("name in ({})".format(joined_location_names))
+
+ locations = session.query(
+ "select id, name from Location where {}".format(
+ " or ".join(query_filters)
+ )
+ ).all()
+ # Fill locations in components
+ for location in locations:
+ location_id = location["id"]
+ location_name = location["name"]
+ if location_id in components_by_location_id:
+ for component in components_by_location_id[location_id]:
+ component["component_location"] = location
+
+ if location_name in components_by_location_name:
+ for component in components_by_location_name[location_name]:
+ component["component_location"] = location
+
+ def _ensure_asset_types_exists(self, session, component_list):
+ """Make sure that all AssetType entities exists for integration.
+
+ Returns:
+ dict: All asset types by short name.
+ """
+ # Query existing asset types
+ asset_types = session.query("select id, short from AssetType").all()
+ # Stpore all existing short names
+ asset_type_shorts = {asset_type["short"] for asset_type in asset_types}
+ # Check which asset types are missing and store them
+ asset_type_names_by_missing_shorts = {}
+ default_short_name = "upload"
+ for data in component_list:
+ asset_type_data = data.get("assettype_data") or {}
+ asset_type_short = asset_type_data.get("short")
+ if not asset_type_short:
+ # Use default asset type name if not set and change the
+ # input data
+ asset_type_short = default_short_name
+ asset_type_data["short"] = asset_type_short
+ data["assettype_data"] = asset_type_data
+
+ if (
+ # Skip if short name exists
+ asset_type_short in asset_type_shorts
+ # Skip if short name was already added to missing types
+ # and asset type name is filled
+ # - if asset type name is missing then try use name from other
+ # data
+ or asset_type_names_by_missing_shorts.get(asset_type_short)
+ ):
+ continue
+
+ asset_type_names_by_missing_shorts[asset_type_short] = (
+ asset_type_data.get("name")
+ )
+
+ # Create missing asset types if there are any
+ if asset_type_names_by_missing_shorts:
+ self.log.info("Creating asset types with short names: {}".format(
+ ", ".join(asset_type_names_by_missing_shorts.keys())
+ ))
+ for missing_short, type_name in (
+ asset_type_names_by_missing_shorts.items()
+ ):
+ # Use short for name if name is not defined
+ if not type_name:
+ type_name = missing_short
+ # Use short name also for name
+ # - there is not other source for 'name'
+ session.create(
+ "AssetType",
+ {
+ "short": missing_short,
+ "name": type_name
+ }
+ )
+
+ # Commit creation
+ session.commit()
+ # Requery asset types
+ asset_types = session.query(
+ "select id, short from AssetType"
+ ).all()
+
+ return {asset_type["short"]: asset_type for asset_type in asset_types}
+
+ def _ensure_asset_exists(
+ self, session, asset_data, asset_type_id, parent_id
+ ):
+ asset_name = asset_data["name"]
+ asset_entity = self._query_asset(
+ session, asset_name, asset_type_id, parent_id
+ )
+ if asset_entity is not None:
+ return asset_entity
+
+ asset_data = {
+ "name": asset_name,
+ "type_id": asset_type_id,
+ "context_id": parent_id
+ }
+ self.log.info("Created new Asset with data: {}.".format(asset_data))
+ session.create("Asset", asset_data)
+ session.commit()
+ return self._query_asset(session, asset_name, asset_type_id, parent_id)
+
+ def _query_asset(self, session, asset_name, asset_type_id, parent_id):
+ return session.query(
+ (
+ "select id from Asset"
+ " where name is \"{}\""
+ " and type_id is \"{}\""
+ " and context_id is \"{}\""
+ ).format(asset_name, asset_type_id, parent_id)
+ ).first()
+
+ def _ensure_asset_version_exists(
+ self,
+ session,
+ asset_version_data,
+ asset_id,
+ task_entity,
+ status_ids_by_name
+ ):
+ task_id = None
+ if task_entity:
+ task_id = task_entity["id"]
+
+ status_name = asset_version_data.pop("status_name", None)
+
+ # Try query asset version by criteria (asset id and version)
+ version = asset_version_data.get("version") or 0
+ asset_version_entity = self._query_asset_version(
+ session, version, asset_id
+ )
+
+ # Prepare comment value
+ comment = asset_version_data.get("comment") or ""
+ if asset_version_entity is not None:
+ changed = False
+ if comment != asset_version_entity["comment"]:
+ asset_version_entity["comment"] = comment
+ changed = True
+
+ if task_id != asset_version_entity["task_id"]:
+ asset_version_entity["task_id"] = task_id
+ changed = True
+
+ if changed:
+ session.commit()
+
+ else:
+ new_asset_version_data = {
+ "version": version,
+ "asset_id": asset_id
+ }
+ if task_id:
+ new_asset_version_data["task_id"] = task_id
+
+ if comment:
+ new_asset_version_data["comment"] = comment
+
+ self.log.info("Created new AssetVersion with data {}".format(
+ new_asset_version_data
+ ))
+ session.create("AssetVersion", new_asset_version_data)
+ session.commit()
+ asset_version_entity = self._query_asset_version(
+ session, version, asset_id
+ )
+
+ if status_name:
+ status_id = status_ids_by_name.get(status_name.lower())
+ if not status_id:
+ self.log.info((
+ "Ftrack status with name \"{}\""
+ " for AssetVersion was not found."
+ ).format(status_name))
+
+ elif asset_version_entity["status_id"] != status_id:
+ asset_version_entity["status_id"] = status_id
+ session.commit()
+
+ # Set custom attributes if there were any set
+ custom_attrs = asset_version_data.get("custom_attributes") or {}
+ for attr_key, attr_value in custom_attrs.items():
+ if attr_key in asset_version_entity["custom_attributes"]:
+ try:
+ asset_version_entity["custom_attributes"][attr_key] = (
+ attr_value
+ )
+ session.commit()
+ continue
+ except Exception:
+ session.rollback()
+ session._configure_locations()
+
+ self.log.warning(
+ (
+ "Custom Attrubute \"{0}\" is not available for"
+ " AssetVersion <{1}>. Can't set it's value to: \"{2}\""
+ ).format(
+ attr_key, asset_version_entity["id"], str(attr_value)
+ )
+ )
+
+ return asset_version_entity
+
+ def _query_asset_version(self, session, version, asset_id):
+ return session.query(
+ (
+ "select id, task_id, comment from AssetVersion"
+ " where version is \"{}\" and asset_id is \"{}\""
+ ).format(version, asset_id)
+ ).first()
+
+ def create_component(self, session, asset_version_entity, data):
+ component_data = data.get("component_data") or {}
+
+ if not component_data.get("name"):
+ component_data["name"] = "main"
+
+ version_id = asset_version_entity["id"]
+ component_data["version_id"] = version_id
+ component_entity = session.query(
+ (
+ "select id, name from Component where name is \"{}\""
+ " and version_id is \"{}\""
+ ).format(component_data["name"], version_id)
+ ).first()
+
+ component_overwrite = data.get("component_overwrite", False)
+ location = data.get("component_location", session.pick_location())
+
+ # Overwrite existing component data if requested.
+ if component_entity and component_overwrite:
+ origin_location = session.query(
+ "Location where name is \"ftrack.origin\""
+ ).one()
+
+ # Removing existing members from location
+ components = list(component_entity.get("members", []))
+ components += [component_entity]
+ for component in components:
+ for loc in component["component_locations"]:
+ if location["id"] == loc["location_id"]:
+ location.remove_component(
+ component, recursive=False
+ )
+
+ # Deleting existing members on component entity
+ for member in component_entity.get("members", []):
+ session.delete(member)
+ del(member)
+
+ try:
+ session.commit()
+ except Exception:
+ tp, value, tb = sys.exc_info()
+ session.rollback()
+ session._configure_locations()
+ six.reraise(tp, value, tb)
+
+ # Reset members in memory
+ if "members" in component_entity.keys():
+ component_entity["members"] = []
+
+ # Add components to origin location
+ try:
+ collection = clique.parse(data["component_path"])
+ except ValueError:
+ # Assume its a single file
+ # Changing file type
+ name, ext = os.path.splitext(data["component_path"])
+ component_entity["file_type"] = ext
+
+ origin_location.add_component(
+ component_entity, data["component_path"]
+ )
+ else:
+ # Changing file type
+ component_entity["file_type"] = collection.format("{tail}")
+
+ # Create member components for sequence.
+ for member_path in collection:
+
+ size = 0
+ try:
+ size = os.path.getsize(member_path)
+ except OSError:
+ pass
+
+ name = collection.match(member_path).group("index")
+
+ member_data = {
+ "name": name,
+ "container": component_entity,
+ "size": size,
+ "file_type": os.path.splitext(member_path)[-1]
+ }
+
+ component = session.create(
+ "FileComponent", member_data
+ )
+ origin_location.add_component(
+ component, member_path, recursive=False
+ )
+ component_entity["members"].append(component)
+
+ # Add components to location.
+ location.add_component(
+ component_entity, origin_location, recursive=True
+ )
+
+ data["component"] = component_entity
+ self.log.info(
+ (
+ "Overwriting Component with path: {0}, data: {1},"
+ " location: {2}"
+ ).format(
+ data["component_path"],
+ component_data,
+ location
+ )
+ )
+
+ # Extracting metadata, and adding after entity creation. This is
+ # due to a ftrack_api bug where you can't add metadata on creation.
+ component_metadata = component_data.pop("metadata", {})
+
+ # Create new component if none exists.
+ new_component = False
+ if not component_entity:
+ component_entity = asset_version_entity.create_component(
+ data["component_path"],
+ data=component_data,
+ location=location
+ )
+ data["component"] = component_entity
+ self.log.info(
+ (
+ "Created new Component with path: {0}, data: {1},"
+ " metadata: {2}, location: {3}"
+ ).format(
+ data["component_path"],
+ component_data,
+ component_metadata,
+ location
+ )
+ )
+ new_component = True
+
+ # Adding metadata
+ existing_component_metadata = component_entity["metadata"]
+ existing_component_metadata.update(component_metadata)
+ component_entity["metadata"] = existing_component_metadata
+
+ # if component_data['name'] = 'ftrackreview-mp4-mp4':
+ # assetversion_entity["thumbnail_id"]
+
+ # Setting assetversion thumbnail
+ if data.get("thumbnail"):
+ asset_version_entity["thumbnail_id"] = component_entity["id"]
+
+ # Inform user about no changes to the database.
+ if (
+ component_entity
+ and not component_overwrite
+ and not new_component
+ ):
+ data["component"] = component_entity
+ self.log.info(
+ "Found existing component, and no request to overwrite. "
+ "Nothing has been changed."
+ )
+ else:
+ # Commit changes.
+ try:
+ session.commit()
+ except Exception:
+ tp, value, tb = sys.exc_info()
+ session.rollback()
+ session._configure_locations()
+ six.reraise(tp, value, tb)
+
+ def _create_components(self, session, asset_versions_data_by_id):
+ for item in asset_versions_data_by_id.values():
+ asset_version_entity = item["asset_version"]
+ component_items = item["component_items"]
+
+ component_entities = session.query(
+ (
+ "select id, name from Component where version_id is \"{}\""
+ ).format(asset_version_entity["id"])
+ ).all()
+
+ existing_component_names = {
+ component["name"]
+ for component in component_entities
+ }
+
+ contain_review = "ftrackreview-mp4" in existing_component_names
+ thumbnail_component_item = None
+ for component_item in component_items:
+ component_data = component_item.get("component_data") or {}
+ component_name = component_data.get("name")
+ if component_name == "ftrackreview-mp4":
+ contain_review = True
+ elif component_name == "ftrackreview-image":
+ thumbnail_component_item = component_item
+
+ if contain_review and thumbnail_component_item:
+ thumbnail_component_item["component_data"]["name"] = (
+ "thumbnail"
+ )
+
+ # Component
+ for component_item in component_items:
+ self.create_component(
+ session, asset_version_entity, component_item
+ )
diff --git a/openpype/modules/default_modules/ftrack/plugins/publish/integrate_ftrack_component_overwrite.py b/openpype/modules/ftrack/plugins/publish/integrate_ftrack_component_overwrite.py
similarity index 76%
rename from openpype/modules/default_modules/ftrack/plugins/publish/integrate_ftrack_component_overwrite.py
rename to openpype/modules/ftrack/plugins/publish/integrate_ftrack_component_overwrite.py
index 047fd8462ca..8cb23363917 100644
--- a/openpype/modules/default_modules/ftrack/plugins/publish/integrate_ftrack_component_overwrite.py
+++ b/openpype/modules/ftrack/plugins/publish/integrate_ftrack_component_overwrite.py
@@ -13,7 +13,10 @@ class IntegrateFtrackComponentOverwrite(pyblish.api.InstancePlugin):
active = False
def process(self, instance):
- component_list = instance.data['ftrackComponentsList']
+ component_list = instance.data.get('ftrackComponentsList')
+ if not component_list:
+ self.log.info("No component to overwrite...")
+ return
for cl in component_list:
cl['component_overwrite'] = True
diff --git a/openpype/modules/ftrack/plugins/publish/integrate_ftrack_description.py b/openpype/modules/ftrack/plugins/publish/integrate_ftrack_description.py
new file mode 100644
index 00000000000..e7c265988ee
--- /dev/null
+++ b/openpype/modules/ftrack/plugins/publish/integrate_ftrack_description.py
@@ -0,0 +1,113 @@
+"""
+Requires:
+ context > comment
+ context > ftrackSession
+ instance > ftrackIntegratedAssetVersionsData
+"""
+
+import sys
+import json
+
+import six
+import pyblish.api
+from openpype.lib import StringTemplate
+
+
+class IntegrateFtrackDescription(pyblish.api.InstancePlugin):
+ """Add description to AssetVersions in Ftrack."""
+
+ # Must be after integrate asset new
+ order = pyblish.api.IntegratorOrder + 0.4999
+ label = "Integrate Ftrack description"
+ families = ["ftrack"]
+ optional = True
+
+ # Can be set in settings:
+ # - Allows `intent` and `comment` keys
+ description_template = "{comment}"
+
+ def process(self, instance):
+ if not self.description_template:
+ self.log.info("Skipping. Description template is not set.")
+ return
+
+ # Check if there are any integrated AssetVersion entities
+ asset_versions_key = "ftrackIntegratedAssetVersionsData"
+ asset_versions_data_by_id = instance.data.get(asset_versions_key)
+ if not asset_versions_data_by_id:
+ self.log.info("There are any integrated AssetVersions")
+ return
+
+ comment = (instance.context.data.get("comment") or "").strip()
+ if not comment:
+ self.log.info("Comment is not set.")
+ else:
+ self.log.debug("Comment is set to `{}`".format(comment))
+
+ intent = instance.context.data.get("intent")
+ if intent and "{intent}" in self.description_template:
+ value = intent.get("value")
+ if value:
+ intent = intent.get("label") or value
+
+ if not intent and not comment:
+ self.log.info("Skipping. Intent and comment are empty.")
+ return
+
+ # if intent label is set then format comment
+ # - it is possible that intent_label is equal to "" (empty string)
+ if intent:
+ self.log.debug("Intent is set to `{}`.".format(intent))
+ else:
+ self.log.debug("Intent is not set.")
+
+ # If we would like to use more "optional" possibilities we would have
+ # come up with some expressions in templates or speicifc templates
+ # for all 3 possible combinations when comment and intent are
+ # set or not (when both are not set then description does not
+ # make sense).
+ fill_data = {}
+ if comment:
+ fill_data["comment"] = comment
+ if intent:
+ fill_data["intent"] = intent
+
+ description = StringTemplate.format_template(
+ self.description_template, fill_data
+ )
+ if not description.solved:
+ self.log.warning((
+ "Couldn't solve template \"{}\" with data {}"
+ ).format(
+ self.description_template, json.dumps(fill_data, indent=4)
+ ))
+ return
+
+ if not description:
+ self.log.debug((
+ "Skipping. Result of template is empty string."
+ " Template \"{}\" Fill data: {}"
+ ).format(
+ self.description_template, json.dumps(fill_data, indent=4)
+ ))
+ return
+
+ session = instance.context.data["ftrackSession"]
+ for asset_version_data in asset_versions_data_by_id.values():
+ asset_version = asset_version_data["asset_version"]
+
+ # Backwards compatibility for older settings using
+ # attribute 'note_with_intent_template'
+
+ asset_version["comment"] = description
+
+ try:
+ session.commit()
+ self.log.debug("Comment added to AssetVersion \"{}\"".format(
+ str(asset_version)
+ ))
+ except Exception:
+ tp, value, tb = sys.exc_info()
+ session.rollback()
+ session._configure_locations()
+ six.reraise(tp, value, tb)
diff --git a/openpype/modules/ftrack/plugins/publish/integrate_ftrack_farm_status.py b/openpype/modules/ftrack/plugins/publish/integrate_ftrack_farm_status.py
new file mode 100644
index 00000000000..ab5738c33f5
--- /dev/null
+++ b/openpype/modules/ftrack/plugins/publish/integrate_ftrack_farm_status.py
@@ -0,0 +1,150 @@
+import pyblish.api
+from openpype.lib import filter_profiles
+
+
+class IntegrateFtrackFarmStatus(pyblish.api.ContextPlugin):
+ """Change task status when should be published on farm.
+
+ Instance which has set "farm" key in data to 'True' is considered as will
+ be rendered on farm thus it's status should be changed.
+ """
+
+ order = pyblish.api.IntegratorOrder + 0.48
+ label = "Integrate Ftrack Farm Status"
+
+ farm_status_profiles = []
+
+ def process(self, context):
+ # Quick end
+ if not self.farm_status_profiles:
+ project_name = context.data["projectName"]
+ self.log.info((
+ "Status profiles are not filled for project \"{}\". Skipping"
+ ).format(project_name))
+ return
+
+ filtered_instances = self.filter_instances(context)
+ instances_with_status_names = self.get_instances_with_statuse_names(
+ context, filtered_instances
+ )
+ if instances_with_status_names:
+ self.fill_statuses(context, instances_with_status_names)
+
+ def filter_instances(self, context):
+ filtered_instances = []
+ for instance in context:
+ # Skip disabled instances
+ if instance.data.get("publish") is False:
+ continue
+ subset_name = instance.data["subset"]
+ msg_start = "Skipping instance {}.".format(subset_name)
+ if not instance.data.get("farm"):
+ self.log.debug(
+ "{} Won't be rendered on farm.".format(msg_start)
+ )
+ continue
+
+ task_entity = instance.data.get("ftrackTask")
+ if not task_entity:
+ self.log.debug(
+ "{} Does not have filled task".format(msg_start)
+ )
+ continue
+
+ filtered_instances.append(instance)
+ return filtered_instances
+
+ def get_instances_with_statuse_names(self, context, instances):
+ instances_with_status_names = []
+ for instance in instances:
+ family = instance.data["family"]
+ subset_name = instance.data["subset"]
+ task_entity = instance.data["ftrackTask"]
+ host_name = context.data["hostName"]
+ task_name = task_entity["name"]
+ task_type = task_entity["type"]["name"]
+ status_profile = filter_profiles(
+ self.farm_status_profiles,
+ {
+ "hosts": host_name,
+ "task_types": task_type,
+ "task_names": task_name,
+ "families": family,
+ "subsets": subset_name,
+ },
+ logger=self.log
+ )
+ if not status_profile:
+ # There already is log in 'filter_profiles'
+ continue
+
+ status_name = status_profile["status_name"]
+ if status_name:
+ instances_with_status_names.append((instance, status_name))
+ return instances_with_status_names
+
+ def fill_statuses(self, context, instances_with_status_names):
+ # Prepare available task statuses on the project
+ project_name = context.data["projectName"]
+ session = context.data["ftrackSession"]
+ project_entity = session.query((
+ "select project_schema from Project where full_name is \"{}\""
+ ).format(project_name)).one()
+ project_schema = project_entity["project_schema"]
+
+ task_type_ids = set()
+ for item in instances_with_status_names:
+ instance, _ = item
+ task_entity = instance.data["ftrackTask"]
+ task_type_ids.add(task_entity["type"]["id"])
+
+ task_statuses_by_type_id = {
+ task_type_id: project_schema.get_statuses("Task", task_type_id)
+ for task_type_id in task_type_ids
+ }
+
+ # Keep track if anything has changed
+ skipped_status_names = set()
+ status_changed = False
+ for item in instances_with_status_names:
+ instance, status_name = item
+ task_entity = instance.data["ftrackTask"]
+ task_statuses = task_statuses_by_type_id[task_entity["type"]["id"]]
+ status_name_low = status_name.lower()
+
+ status_id = None
+ status_name = None
+ # Skip if status name was already tried to be found
+ for status in task_statuses:
+ if status["name"].lower() == status_name_low:
+ status_id = status["id"]
+ status_name = status["name"]
+ break
+
+ if status_id is None:
+ if status_name_low not in skipped_status_names:
+ skipped_status_names.add(status_name_low)
+ joined_status_names = ", ".join({
+ '"{}"'.format(status["name"])
+ for status in task_statuses
+ })
+ self.log.warning((
+ "Status \"{}\" is not available on project \"{}\"."
+ " Available statuses are {}"
+ ).format(status_name, project_name, joined_status_names))
+ continue
+
+ # Change task status id
+ if status_id != task_entity["status_id"]:
+ task_entity["status_id"] = status_id
+ status_changed = True
+ path = "/".join([
+ item["name"]
+ for item in task_entity["link"]
+ ])
+ self.log.debug("Set status \"{}\" to \"{}\"".format(
+ status_name, path
+ ))
+
+ if status_changed:
+ session.commit()
diff --git a/openpype/modules/ftrack/plugins/publish/integrate_ftrack_instances.py b/openpype/modules/ftrack/plugins/publish/integrate_ftrack_instances.py
new file mode 100644
index 00000000000..5ff75e7060b
--- /dev/null
+++ b/openpype/modules/ftrack/plugins/publish/integrate_ftrack_instances.py
@@ -0,0 +1,556 @@
+import os
+import json
+import copy
+import pyblish.api
+
+from openpype.lib.openpype_version import get_openpype_version
+from openpype.lib.transcoding import (
+ get_ffprobe_streams,
+ convert_ffprobe_fps_to_float,
+)
+from openpype.lib.profiles_filtering import filter_profiles
+
+
+class IntegrateFtrackInstance(pyblish.api.InstancePlugin):
+ """Collect ftrack component data (not integrate yet).
+
+ Add ftrack component list to instance.
+ """
+
+ order = pyblish.api.IntegratorOrder + 0.48
+ label = "Integrate Ftrack Component"
+ families = ["ftrack"]
+
+ metadata_keys_to_label = {
+ "openpype_version": "OpenPype version",
+ "frame_start": "Frame start",
+ "frame_end": "Frame end",
+ "duration": "Duration",
+ "width": "Resolution width",
+ "height": "Resolution height",
+ "fps": "FPS",
+ "codec": "Codec"
+ }
+
+ family_mapping = {
+ "camera": "cam",
+ "look": "look",
+ "mayaAscii": "scene",
+ "model": "geo",
+ "rig": "rig",
+ "setdress": "setdress",
+ "pointcache": "cache",
+ "render": "render",
+ "prerender": "render",
+ "render2d": "render",
+ "nukescript": "comp",
+ "write": "render",
+ "review": "mov",
+ "plate": "img",
+ "audio": "audio",
+ "workfile": "scene",
+ "animation": "cache",
+ "image": "img",
+ "reference": "reference"
+ }
+ keep_first_subset_name_for_review = True
+ asset_versions_status_profiles = []
+ additional_metadata_keys = []
+
+ def process(self, instance):
+ self.log.debug("instance {}".format(instance))
+
+ instance_repres = instance.data.get("representations")
+ if not instance_repres:
+ self.log.info((
+ "Skipping instance. Does not have any representations {}"
+ ).format(str(instance)))
+ return
+
+ instance_version = instance.data.get("version")
+ if instance_version is None:
+ raise ValueError("Instance version not set")
+
+ version_number = int(instance_version)
+
+ family = instance.data["family"]
+
+ # Perform case-insensitive family mapping
+ family_low = family.lower()
+ asset_type = instance.data.get("ftrackFamily")
+ if not asset_type:
+ for map_family, map_value in self.family_mapping.items():
+ if map_family.lower() == family_low:
+ asset_type = map_value
+ break
+
+ if not asset_type:
+ asset_type = "upload"
+
+ self.log.debug(
+ "Family: {}\nMapping: {}".format(family_low, self.family_mapping)
+ )
+ status_name = self._get_asset_version_status_name(instance)
+
+ # Base of component item data
+ # - create a copy of this object when want to use it
+ base_component_item = {
+ "assettype_data": {
+ "short": asset_type,
+ },
+ "asset_data": {
+ "name": instance.data["subset"],
+ },
+ "assetversion_data": {
+ "version": version_number,
+ "comment": instance.context.data.get("comment") or "",
+ "status_name": status_name
+ },
+ "component_overwrite": False,
+ # This can be change optionally
+ "thumbnail": False,
+ # These must be changed for each component
+ "component_data": None,
+ "component_path": None,
+ "component_location": None,
+ "component_location_name": None,
+ "additional_data": {}
+ }
+
+ # Filter types of representations
+ review_representations = []
+ thumbnail_representations = []
+ other_representations = []
+ for repre in instance_repres:
+ self.log.debug("Representation {}".format(repre))
+ repre_tags = repre.get("tags") or []
+ if repre.get("thumbnail") or "thumbnail" in repre_tags:
+ thumbnail_representations.append(repre)
+
+ elif "ftrackreview" in repre_tags:
+ review_representations.append(repre)
+
+ else:
+ other_representations.append(repre)
+
+ # Prepare ftrack locations
+ unmanaged_location_name = "ftrack.unmanaged"
+ ftrack_server_location_name = "ftrack.server"
+
+ # Components data
+ component_list = []
+ # Components that will be duplicated to unmanaged location
+ src_components_to_add = []
+
+ # Create thumbnail components
+ # TODO what if there is multiple thumbnails?
+ first_thumbnail_component = None
+ first_thumbnail_component_repre = None
+ for repre in thumbnail_representations:
+ repre_path = self._get_repre_path(instance, repre, False)
+ if not repre_path:
+ self.log.warning(
+ "Published path is not set and source was removed."
+ )
+ continue
+
+ # Create copy of base comp item and append it
+ thumbnail_item = copy.deepcopy(base_component_item)
+ thumbnail_item["component_path"] = repre_path
+ thumbnail_item["component_data"] = {
+ "name": "thumbnail"
+ }
+ thumbnail_item["thumbnail"] = True
+
+ # Create copy of item before setting location
+ src_components_to_add.append(copy.deepcopy(thumbnail_item))
+ # Create copy of first thumbnail
+ if first_thumbnail_component is None:
+ first_thumbnail_component_repre = repre
+ first_thumbnail_component = thumbnail_item
+ # Set location
+ thumbnail_item["component_location_name"] = (
+ ftrack_server_location_name
+ )
+
+ # Add item to component list
+ component_list.append(thumbnail_item)
+
+ if first_thumbnail_component is not None:
+ width = first_thumbnail_component_repre.get("width")
+ height = first_thumbnail_component_repre.get("height")
+ if not width or not height:
+ component_path = first_thumbnail_component["component_path"]
+ streams = []
+ try:
+ streams = get_ffprobe_streams(component_path)
+ except Exception:
+ self.log.debug((
+ "Failed to retrieve information about intput {}"
+ ).format(component_path))
+
+ for stream in streams:
+ if "width" in stream and "height" in stream:
+ width = stream["width"]
+ height = stream["height"]
+ break
+
+ if width and height:
+ component_data = first_thumbnail_component["component_data"]
+ component_data["name"] = "ftrackreview-image"
+ component_data["metadata"] = {
+ "ftr_meta": json.dumps({
+ "width": width,
+ "height": height,
+ "format": "image"
+ })
+ }
+
+ # Create review components
+ # Change asset name of each new component for review
+ is_first_review_repre = True
+ not_first_components = []
+ extended_asset_name = ""
+ multiple_reviewable = len(review_representations) > 1
+ for repre in review_representations:
+ repre_path = self._get_repre_path(instance, repre, False)
+ if not repre_path:
+ self.log.warning(
+ "Published path is not set and source was removed."
+ )
+ continue
+
+ # Create copy of base comp item and append it
+ review_item = copy.deepcopy(base_component_item)
+
+ # get asset name and define extended name variant
+ asset_name = review_item["asset_data"]["name"]
+ extended_asset_name = "_".join(
+ (asset_name, repre["name"])
+ )
+
+ # reset extended if no need for extended asset name
+ if (
+ self.keep_first_subset_name_for_review
+ and is_first_review_repre
+ ):
+ extended_asset_name = ""
+ else:
+ # only rename if multiple reviewable
+ if multiple_reviewable:
+ review_item["asset_data"]["name"] = extended_asset_name
+ else:
+ extended_asset_name = ""
+
+ # rename all already created components
+ # only if first repre and extended name available
+ if is_first_review_repre and extended_asset_name:
+ # and rename all already created components
+ for _ci in component_list:
+ _ci["asset_data"]["name"] = extended_asset_name
+
+ # and rename all already created src components
+ for _sci in src_components_to_add:
+ _sci["asset_data"]["name"] = extended_asset_name
+
+ # rename also first thumbnail component if any
+ if first_thumbnail_component is not None:
+ first_thumbnail_component[
+ "asset_data"]["name"] = extended_asset_name
+
+ # Change location
+ review_item["component_path"] = repre_path
+ # Change component data
+ review_item["component_data"] = {
+ # Default component name is "main".
+ "name": "ftrackreview-mp4",
+ "metadata": self._prepare_component_metadata(
+ instance, repre, repre_path, True
+ )
+ }
+
+ if is_first_review_repre:
+ is_first_review_repre = False
+ else:
+ # later detection for thumbnail duplication
+ not_first_components.append(review_item)
+
+ # Create copy of item before setting location
+ src_components_to_add.append(copy.deepcopy(review_item))
+
+ # Set location
+ review_item["component_location_name"] = (
+ ftrack_server_location_name
+ )
+ # Add item to component list
+ component_list.append(review_item)
+
+ # Duplicate thumbnail component for all not first reviews
+ if first_thumbnail_component is not None:
+ for component_item in not_first_components:
+ asset_name = component_item["asset_data"]["name"]
+ new_thumbnail_component = copy.deepcopy(
+ first_thumbnail_component
+ )
+ new_thumbnail_component["asset_data"]["name"] = asset_name
+ new_thumbnail_component["component_location_name"] = (
+ ftrack_server_location_name
+ )
+ component_list.append(new_thumbnail_component)
+
+ # Add source components for review and thubmnail components
+ for copy_src_item in src_components_to_add:
+ # Make sure thumbnail is disabled
+ copy_src_item["thumbnail"] = False
+ # Set location
+ copy_src_item["component_location_name"] = unmanaged_location_name
+ # Modify name of component to have suffix "_src"
+ component_data = copy_src_item["component_data"]
+ component_name = component_data["name"]
+ component_data["name"] = component_name + "_src"
+ component_data["metadata"] = self._prepare_component_metadata(
+ instance, repre, copy_src_item["component_path"], False
+ )
+ component_list.append(copy_src_item)
+
+ # Add others representations as component
+ for repre in other_representations:
+ published_path = self._get_repre_path(instance, repre, True)
+ if not published_path:
+ continue
+ # Create copy of base comp item and append it
+ other_item = copy.deepcopy(base_component_item)
+
+ # add extended name if any
+ if (
+ not self.keep_first_subset_name_for_review
+ and extended_asset_name
+ ):
+ other_item["asset_data"]["name"] = extended_asset_name
+
+ component_data = {
+ "name": repre["name"],
+ "metadata": self._prepare_component_metadata(
+ instance, repre, published_path, False
+ )
+ }
+ other_item["component_data"] = component_data
+ other_item["component_location_name"] = unmanaged_location_name
+ other_item["component_path"] = published_path
+ component_list.append(other_item)
+
+ def json_obj_parser(obj):
+ return str(obj)
+
+ self.log.debug("Components list: {}".format(
+ json.dumps(
+ component_list,
+ sort_keys=True,
+ indent=4,
+ default=json_obj_parser
+ )
+ ))
+ instance.data["ftrackComponentsList"] = component_list
+
+ def _collect_additional_metadata(self, streams):
+ pass
+
+ def _get_repre_path(self, instance, repre, only_published):
+ """Get representation path that can be used for integration.
+
+ When 'only_published' is set to true the validation of path is not
+ relevant. In that case we just need what is set in 'published_path'
+ as "reference". The reference is not used to get or upload the file but
+ for reference where the file was published.
+
+ Args:
+ instance (pyblish.Instance): Processed instance object. Used
+ for source of staging dir if representation does not have
+ filled it.
+ repre (dict): Representation on instance which could be and
+ could not be integrated with main integrator.
+ only_published (bool): Care only about published paths and
+ ignore if filepath is not existing anymore.
+
+ Returns:
+ str: Path to representation file.
+ None: Path is not filled or does not exists.
+ """
+
+ published_path = repre.get("published_path")
+ if published_path:
+ published_path = os.path.normpath(published_path)
+ if os.path.exists(published_path):
+ return published_path
+
+ if only_published:
+ return published_path
+
+ comp_files = repre["files"]
+ if isinstance(comp_files, (tuple, list, set)):
+ filename = comp_files[0]
+ else:
+ filename = comp_files
+
+ staging_dir = repre.get("stagingDir")
+ if not staging_dir:
+ staging_dir = instance.data["stagingDir"]
+ src_path = os.path.normpath(os.path.join(staging_dir, filename))
+ if os.path.exists(src_path):
+ return src_path
+ return None
+
+ def _get_asset_version_status_name(self, instance):
+ if not self.asset_versions_status_profiles:
+ return None
+
+ # Prepare filtering data for new asset version status
+ anatomy_data = instance.data["anatomyData"]
+ task_type = anatomy_data.get("task", {}).get("type")
+ filtering_criteria = {
+ "families": instance.data["family"],
+ "hosts": instance.context.data["hostName"],
+ "task_types": task_type
+ }
+ matching_profile = filter_profiles(
+ self.asset_versions_status_profiles,
+ filtering_criteria
+ )
+ if not matching_profile:
+ return None
+
+ return matching_profile["status"] or None
+
+ def _prepare_component_metadata(
+ self, instance, repre, component_path, is_review
+ ):
+ metadata = {}
+ if "openpype_version" in self.additional_metadata_keys:
+ label = self.metadata_keys_to_label["openpype_version"]
+ metadata[label] = get_openpype_version()
+
+ extension = os.path.splitext(component_path)[-1]
+ streams = []
+ try:
+ streams = get_ffprobe_streams(component_path)
+ except Exception:
+ self.log.debug((
+ "Failed to retrieve information about intput {}"
+ ).format(component_path))
+
+ # Find video streams
+ video_streams = [
+ stream
+ for stream in streams
+ if stream["codec_type"] == "video"
+ ]
+ # Skip if there are not video streams
+ # - exr is special case which can have issues with reading through
+ # ffmpegh but we want to set fps for it
+ if not video_streams and extension not in [".exr"]:
+ return metadata
+
+ stream_width = None
+ stream_height = None
+ stream_fps = None
+ frame_out = None
+ codec_label = None
+ for video_stream in video_streams:
+ codec_label = video_stream.get("codec_long_name")
+ if not codec_label:
+ codec_label = video_stream.get("codec")
+
+ if codec_label:
+ pix_fmt = video_stream.get("pix_fmt")
+ if pix_fmt:
+ codec_label += " ({})".format(pix_fmt)
+
+ tmp_width = video_stream.get("width")
+ tmp_height = video_stream.get("height")
+ if tmp_width and tmp_height:
+ stream_width = tmp_width
+ stream_height = tmp_height
+
+ input_framerate = video_stream.get("r_frame_rate")
+ stream_duration = video_stream.get("duration")
+ if input_framerate is None or stream_duration is None:
+ continue
+ try:
+ stream_fps = convert_ffprobe_fps_to_float(
+ input_framerate
+ )
+ except ValueError:
+ self.log.warning((
+ "Could not convert ffprobe fps to float \"{}\""
+ ).format(input_framerate))
+ continue
+
+ stream_width = tmp_width
+ stream_height = tmp_height
+
+ self.log.debug("FPS from stream is {} and duration is {}".format(
+ input_framerate, stream_duration
+ ))
+ frame_out = float(stream_duration) * stream_fps
+ break
+
+ # Prepare FPS
+ instance_fps = instance.data.get("fps")
+ if instance_fps is None:
+ instance_fps = instance.context.data["fps"]
+
+ repre_fps = repre.get("fps")
+ if repre_fps is not None:
+ repre_fps = float(repre_fps)
+
+ fps = stream_fps or repre_fps or instance_fps
+
+ # Prepare frame ranges
+ frame_start = repre.get("frameStartFtrack")
+ frame_end = repre.get("frameEndFtrack")
+ if frame_start is None or frame_end is None:
+ frame_start = instance.data["frameStart"]
+ frame_end = instance.data["frameEnd"]
+ duration = (frame_end - frame_start) + 1
+
+ for key, value in [
+ ("fps", fps),
+ ("frame_start", frame_start),
+ ("frame_end", frame_end),
+ ("duration", duration),
+ ("width", stream_width),
+ ("height", stream_height),
+ ("fps", fps),
+ ("codec", codec_label)
+ ]:
+ if not value or key not in self.additional_metadata_keys:
+ continue
+ label = self.metadata_keys_to_label[key]
+ metadata[label] = value
+
+ if not is_review:
+ ftr_meta = {}
+ if fps:
+ ftr_meta["frameRate"] = fps
+
+ if stream_width and stream_height:
+ ftr_meta["width"] = int(stream_width)
+ ftr_meta["height"] = int(stream_height)
+ metadata["ftr_meta"] = json.dumps(ftr_meta)
+ return metadata
+
+ # Frame end of uploaded video file should be duration in frames
+ # - frame start is always 0
+ # - frame end is duration in frames
+ if not frame_out:
+ frame_out = duration
+
+ # Ftrack documentation says that it is required to have
+ # 'width' and 'height' in review component. But with those values
+ # review video does not play.
+ metadata["ftr_meta"] = json.dumps({
+ "frameIn": 0,
+ "frameOut": frame_out,
+ "frameRate": float(fps)
+ })
+ return metadata
diff --git a/openpype/modules/ftrack/plugins/publish/integrate_ftrack_note.py b/openpype/modules/ftrack/plugins/publish/integrate_ftrack_note.py
new file mode 100644
index 00000000000..ac3fa874e00
--- /dev/null
+++ b/openpype/modules/ftrack/plugins/publish/integrate_ftrack_note.py
@@ -0,0 +1,153 @@
+"""
+Requires:
+ context > hostName
+ context > appName
+ context > appLabel
+ context > comment
+ context > ftrackSession
+ instance > ftrackIntegratedAssetVersionsData
+"""
+
+import sys
+import copy
+
+import six
+import pyblish.api
+from openpype.lib import StringTemplate
+
+
+class IntegrateFtrackNote(pyblish.api.InstancePlugin):
+ """Create comments in Ftrack."""
+
+ # Must be after integrate asset new
+ order = pyblish.api.IntegratorOrder + 0.4999
+ label = "Integrate Ftrack note"
+ families = ["ftrack"]
+ optional = True
+
+ # Can be set in presets:
+ # - Allows only `intent` and `comment` keys
+ note_template = None
+ # Backwards compatibility
+ note_with_intent_template = "{intent}: {comment}"
+ # - note label must exist in Ftrack
+ note_labels = []
+
+ def process(self, instance):
+ # Check if there are any integrated AssetVersion entities
+ asset_versions_key = "ftrackIntegratedAssetVersionsData"
+ asset_versions_data_by_id = instance.data.get(asset_versions_key)
+ if not asset_versions_data_by_id:
+ self.log.info("There are any integrated AssetVersions")
+ return
+
+ context = instance.context
+ host_name = context.data["hostName"]
+ app_name = context.data["appName"]
+ app_label = context.data["appLabel"]
+ comment = (context.data.get("comment") or "").strip()
+ if not comment:
+ self.log.info("Comment is not set.")
+ else:
+ self.log.debug("Comment is set to `{}`".format(comment))
+
+ session = context.data["ftrackSession"]
+
+ intent = instance.context.data.get("intent")
+ intent_label = None
+ if intent:
+ value = intent["value"]
+ if value:
+ intent_label = intent["label"] or value
+
+ # if intent label is set then format comment
+ # - it is possible that intent_label is equal to "" (empty string)
+ if intent_label:
+ self.log.debug(
+ "Intent label is set to `{}`.".format(intent_label)
+ )
+
+ else:
+ self.log.debug("Intent is not set.")
+
+ user = session.query(
+ "User where username is \"{}\"".format(session.api_user)
+ ).first()
+ if not user:
+ self.log.warning(
+ "Was not able to query current User {}".format(
+ session.api_user
+ )
+ )
+
+ labels = []
+ if self.note_labels:
+ all_labels = session.query("select id, name from NoteLabel").all()
+ labels_by_low_name = {lab["name"].lower(): lab for lab in all_labels}
+ for _label in self.note_labels:
+ label = labels_by_low_name.get(_label.lower())
+ if not label:
+ self.log.warning(
+ "Note Label `{}` was not found.".format(_label)
+ )
+ continue
+
+ labels.append(label)
+
+ base_format_data = {
+ "host_name": host_name,
+ "app_name": app_name,
+ "app_label": app_label,
+ "source": instance.data.get("source", '')
+ }
+ if comment:
+ base_format_data["comment"] = comment
+ for asset_version_data in asset_versions_data_by_id.values():
+ asset_version = asset_version_data["asset_version"]
+ component_items = asset_version_data["component_items"]
+
+ published_paths = set()
+ for component_item in component_items:
+ published_paths.add(component_item["component_path"])
+
+ # Backwards compatibility for older settings using
+ # attribute 'note_with_intent_template'
+ template = self.note_template
+ if template is None:
+ template = self.note_with_intent_template
+ format_data = copy.deepcopy(base_format_data)
+ format_data["published_paths"] = "
+
+..code-block:: python
+ >>> before = arrow.get("2018-03-10 23:00:00", "YYYY-MM-DD HH:mm:ss", tzinfo="US/Pacific")
+ >>> after = arrow.get("2018-03-11 04:00:00", "YYYY-MM-DD HH:mm:ss", tzinfo="US/Pacific")
+ >>> result=[(t, t.to("utc")) for t in arrow.Arrow.range("hour", before, after)]
+ >>> for r in result:
+ ... print(r)
+ ...
+ (, )
+ (, )
+ (, )
+ (, )
+ (, )
+
+- [NEW] Added ``humanize`` week granularity translation for Tagalog.
+- [CHANGE] Calls to the ``timestamp`` property now emit a ``DeprecationWarning``. In a future release, ``timestamp`` will be changed to a method to align with Python's datetime module. If you would like to continue using the property, please change your code to use the ``int_timestamp`` or ``float_timestamp`` properties instead.
+- [CHANGE] Expanded and improved Catalan locale.
+- [FIX] Fixed a bug that caused ``Arrow.range()`` to incorrectly cut off ranges in certain scenarios when using month, quarter, or year endings.
+- [FIX] Fixed a bug that caused day of week token parsing to be case sensitive.
+- [INTERNAL] A number of functions were reordered in arrow.py for better organization and grouping of related methods. This change will have no impact on usage.
+- [INTERNAL] A minimum tox version is now enforced for compatibility reasons. Contributors must use tox >3.18.0 going forward.
+
+0.16.0 (2020-08-23)
+-------------------
+
+- [WARN] Arrow will **drop support** for Python 2.7 and 3.5 in the upcoming 1.0.0 release. The 0.16.x and 0.17.x releases are the last to support Python 2.7 and 3.5.
+- [NEW] Implemented `PEP 495
+ >>> before.fold
+ 0
+ >>> before.ambiguous
+ True
+ >>> after = Arrow(2017, 10, 29, 2, 0, tzinfo='Europe/Stockholm', fold=1)
+
+ >>> after = before.replace(fold=1)
+
+
+- [NEW] Added ``normalize_whitespace`` flag to ``arrow.get``. This is useful for parsing log files and/or any files that may contain inconsistent spacing. For example:
+
+.. code-block:: python
+
+ >>> arrow.get("Jun 1 2005 1:33PM", "MMM D YYYY H:mmA", normalize_whitespace=True)
+
+ >>> arrow.get("2013-036 \t 04:05:06Z", normalize_whitespace=True)
+
