How to use is_multipart_upload method in localstack

Best Python code snippet using localstack_python

shotgun.py

Source:shotgun.py

1#!/usr/bin/env python2"""3 -----------------------------------------------------------------------------4 Copyright (c) 2009-2017, Shotgun Software Inc.5 Redistribution and use in source and binary forms, with or without6 modification, are permitted provided that the following conditions are met:7 - Redistributions of source code must retain the above copyright notice, this8 list of conditions and the following disclaimer.9 - Redistributions in binary form must reproduce the above copyright notice,10 this list of conditions and the following disclaimer in the documentation11 and/or other materials provided with the distribution.12 - Neither the name of the Shotgun Software Inc nor the names of its13 contributors may be used to endorse or promote products derived from this14 software without specific prior written permission.15 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"16 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE17 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE18 DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE19 FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL20 DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR21 SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER22 CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,23 OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE24 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.25"""26import base6427import cookielib # used for attachment upload28import cStringIO # used for attachment upload29import datetime30import logging31import mimetools # used for attachment upload32import os33import re34import copy35import stat # used for attachment upload36import sys37import time38import types39import urllib40import urllib2 # used for image upload41import urlparse42import shutil # used for attachment download43# use relative import for versions >=2.5 and package import for python versions <2.544if (sys.version_info[0] > 2) or (sys.version_info[0] == 2 and sys.version_info[1] >= 6):45 from sg_26 import *46elif (sys.version_info[0] > 2) or (sys.version_info[0] == 2 and sys.version_info[1] >= 5):47 from sg_25 import *48else:49 from sg_24 import *50# mimetypes imported in version specific imports51mimetypes.add_type('video/webm','.webm') # webm and mp4 seem to be missing52mimetypes.add_type('video/mp4', '.mp4') # from some OS/distros53LOG = logging.getLogger("shotgun_api3")54"""55Logging instance for shotgun_api356Provides a logging instance where log messages are sent during execution. This instance has no57handler associated with it.58.. seealso:: :ref:`logging`59"""60LOG.setLevel(logging.WARN)61SG_TIMEZONE = SgTimezone()62NO_SSL_VALIDATION = False63"""64Turns off hostname matching validation for SSL certificates65Sometimes there are cases where certificate validation should be disabled. For example, if you66have a self-signed internal certificate that isn't included in our certificate bundle, you may67not require the added security provided by enforcing this.68"""69try:70 import ssl71except ImportError, e:72 if "SHOTGUN_FORCE_CERTIFICATE_VALIDATION" in os.environ:73 raise ImportError("%s. SHOTGUN_FORCE_CERTIFICATE_VALIDATION environment variable prevents "74 "disabling SSL certificate validation." % e)75 LOG.debug("ssl not found, disabling certificate validation")76 NO_SSL_VALIDATION = True77# ----------------------------------------------------------------------------78# Version79__version__ = "3.0.35"80# ----------------------------------------------------------------------------81# Errors82class ShotgunError(Exception):83 """84 Base for all Shotgun API Errors.85 """86 pass87class ShotgunFileDownloadError(ShotgunError):88 """89 Exception for file download-related errors.90 """91 pass92class Fault(ShotgunError):93 """94 Exception when server-side exception detected.95 """96 pass97class AuthenticationFault(Fault):98 """99 Exception when the server side reports an error related to authentication.100 """101 pass102class MissingTwoFactorAuthenticationFault(Fault):103 """104 Exception when the server side reports an error related to missing two-factor authentication105 credentials.106 """107 pass108class UserCredentialsNotAllowedForSSOAuthenticationFault(Fault):109 """110 Exception when the server is configured to use SSO. It is not possible to use111 a username/password pair to authenticate on such server.112 """113 pass114# ----------------------------------------------------------------------------115# API116class ServerCapabilities(object):117 """118 Container for the servers capabilities, such as version enabled features.119 .. warning::120 This class is part of the internal API and its interfaces may change at any time in121 the future. Therefore, usage of this class is discouraged.122 """123 def __init__(self, host, meta):124 """125 ServerCapabilities.__init__126 :param str host: Host name for the server excluding protocol.127 :param dict meta: dict of meta data for the server returned from the info() api method.128 :ivar str host:129 :ivar dict server_info:130 :ivar tuple version: Simple version of the Shotgun server. ``(major, minor, rev)``131 :ivar bool is_dev: ``True`` if server is running a development version of the Shotgun132 codebase.133 """134 # Server host name135 self.host = host136 self.server_info = meta137 # Version from server is major.minor.rev or major.minor.rev."Dev"138 # Store version as tuple and check dev flag139 try:140 self.version = meta.get("version", None)141 except AttributeError:142 self.version = None143 if not self.version:144 raise ShotgunError("The Shotgun Server didn't respond with a version number. "145 "This may be because you are running an older version of "146 "Shotgun against a more recent version of the Shotgun API. "147 "For more information, please contact Shotgun Support.")148 if len(self.version) > 3 and self.version[3] == "Dev":149 self.is_dev = True150 else:151 self.is_dev = False152 self.version = tuple(self.version[:3])153 self._ensure_json_supported()154 def _ensure_support(self, feature, raise_hell=True):155 """156 Checks the server version supports a given feature, raises an exception if it does not.157 :param dict feature: dict where **version** key contains a 3 integer tuple indicating the158 supported server version and **label** key contains a human-readable label str::159 { 'version': (5, 4, 4), 'label': 'project parameter }160 :param bool raise_hell: Whether to raise an exception if the feature is not supported.161 Defaults to ``True``162 :raises: :class:`ShotgunError` if the current server version does not support ``feature``163 """164 if not self.version or self.version < feature['version']:165 if raise_hell:166 raise ShotgunError(167 "%s requires server version %s or higher, "\168 "server is %s" % (feature['label'], _version_str(feature['version']), _version_str(self.version))169 )170 return False171 else:172 return True173 def _ensure_json_supported(self):174 """175 Ensures server has support for JSON API endpoint added in v2.4.0.176 """177 self._ensure_support({178 'version': (2, 4, 0),179 'label': 'JSON API'180 })181 def ensure_include_archived_projects(self):182 """183 Ensures server has support for archived Projects feature added in v5.3.14.184 """185 self._ensure_support({186 'version': (5, 3, 14),187 'label': 'include_archived_projects parameter'188 })189 def ensure_per_project_customization(self):190 """191 Ensures server has support for per-project customization feature added in v5.4.4.192 """193 return self._ensure_support({194 'version': (5, 4, 4),195 'label': 'project parameter'196 }, True)197 def ensure_support_for_additional_filter_presets(self):198 """199 Ensures server has support for additional filter presets feature added in v7.0.0.200 """201 return self._ensure_support({202 'version': (7, 0, 0),203 'label': 'additional_filter_presets parameter'204 }, True)205 def ensure_user_following_support(self):206 """207 Ensures server has support for listing items a user is following, added in v7.0.12.208 """209 return self._ensure_support({210 'version': (7, 0, 12),211 'label': 'user_following parameter'212 }, True)213 def ensure_paging_info_without_counts_support(self):214 """215 Ensures server has support for optimized pagination, added in v7.4.0.216 """217 return self._ensure_support({218 'version': (7, 4, 0),219 'label': 'optimized pagination'220 }, False)221 def ensure_return_image_urls_support(self):222 """223 Ensures server has support for returning thumbnail URLs without additional round-trips, added in v3.3.0.224 """225 return self._ensure_support({226 'version': (3, 3, 0),227 'label': 'return thumbnail URLs'228 }, False)229 def __str__(self):230 return "ServerCapabilities: host %s, version %s, is_dev %s"\231 % (self.host, self.version, self.is_dev)232class ClientCapabilities(object):233 """234 Container for the client capabilities.235 .. warning::236 This class is part of the internal API and its interfaces may change at any time in237 the future. Therefore, usage of this class is discouraged.238 :ivar str platform: The current client platform. Valid values are ``mac``, ``linux``,239 ``windows``, or ``None`` (if the current platform couldn't be determined).240 :ivar str local_path_field: The SG field used for local file paths. This is calculated using241 the value of ``platform``. Ex. ``local_path_mac``.242 :ivar str py_version: Simple version of Python executable as a string. Eg. ``2.7``.243 :ivar str ssl_version: Version of OpenSSL installed. Eg. ``OpenSSL 1.0.2g 1 Mar 2016``. This244 info is only available in Python 2.7+ if the ssl module was imported successfully.245 Defaults to ``unknown``246 """247 def __init__(self):248 system = sys.platform.lower()249 if system == 'darwin':250 self.platform = "mac"251 elif system.startswith('linux'):252 self.platform = 'linux'253 elif system == 'win32':254 self.platform = 'windows'255 else:256 self.platform = None257 if self.platform:258 self.local_path_field = "local_path_%s" % (self.platform)259 else:260 self.local_path_field = None261 self.py_version = ".".join(str(x) for x in sys.version_info[:2])262 # extract the OpenSSL version if we can. The version is only available in Python 2.7 and263 # only if we successfully imported ssl264 self.ssl_version = "unknown"265 try:266 self.ssl_version = ssl.OPENSSL_VERSION267 except (AttributeError, NameError):268 pass269 def __str__(self):270 return "ClientCapabilities: platform %s, local_path_field %s, "\271 "py_verison %s, ssl version %s" % (self.platform, self.local_path_field,272 self.py_version, self.ssl_version)273class _Config(object):274 """275 Container for the client configuration.276 """277 def __init__(self):278 self.max_rpc_attempts = 3279 # From http://docs.python.org/2.6/library/httplib.html:280 # If the optional timeout parameter is given, blocking operations281 # (like connection attempts) will timeout after that many seconds282 # (if it is not given, the global default timeout setting is used)283 self.timeout_secs = None284 self.api_ver = 'api3'285 self.convert_datetimes_to_utc = True286 self.records_per_page = 500287 self.api_key = None288 self.script_name = None289 self.user_login = None290 self.user_password = None291 self.auth_token = None292 self.sudo_as_login = None293 # Authentication parameters to be folded into final auth_params dict294 self.extra_auth_params = None295 # uuid as a string296 self.session_uuid = None297 self.scheme = None298 self.server = None299 self.api_path = None300 # The raw_http_proxy reflects the exact string passed in301 # to the Shotgun constructor. This can be useful if you302 # need to construct a Shotgun API instance based on303 # another Shotgun API instance.304 self.raw_http_proxy = None305 # if a proxy server is being used, the proxy_handler306 # below will contain a urllib2.ProxyHandler instance307 # which can be used whenever a request needs to be made.308 self.proxy_handler = None309 self.proxy_server = None310 self.proxy_port = 8080311 self.proxy_user = None312 self.proxy_pass = None313 self.session_token = None314 self.authorization = None315 self.no_ssl_validation = False316class Shotgun(object):317 """318 Shotgun Client connection.319 """320 # reg ex from321 # http://underground.infovark.com/2008/07/22/iso-date-validation-regex/322 # Note a length check is done before checking the reg ex323 _DATE_PATTERN = re.compile(324 "^(\d{4})\D?(0[1-9]|1[0-2])\D?([12]\d|0[1-9]|3[01])$")325 _DATE_TIME_PATTERN = re.compile(326 "^(\d{4})\D?(0[1-9]|1[0-2])\D?([12]\d|0[1-9]|3[01])"\327 "(\D?([01]\d|2[0-3])\D?([0-5]\d)\D?([0-5]\d)?\D?(\d{3})?)?$")328 _MULTIPART_UPLOAD_CHUNK_SIZE = 20000000329 def __init__(self,330 base_url,331 script_name=None,332 api_key=None,333 convert_datetimes_to_utc=True,334 http_proxy=None,335 ensure_ascii=True,336 connect=True,337 ca_certs=None,338 login=None,339 password=None,340 sudo_as_login=None,341 session_token=None,342 auth_token=None):343 """344 Initializes a new instance of the Shotgun client.345 :param str base_url: http or https url of the Shotgun server. Do not include the trailing346 slash::347 https://example.shotgunstudio.com348 :param str script_name: name of the Script entity used to authenticate to the server.349 If provided, then ``api_key`` must be as well, and neither ``login`` nor ``password``350 can be provided.351 .. seealso:: :ref:`authentication`352 :param str api_key: API key for the provided ``script_name``. Used to authenticate to the353 server. If provided, then ``script_name`` must be as well, and neither ``login`` nor354 ``password`` can be provided.355 .. seealso:: :ref:`authentication`356 :param bool convert_datetimes_to_utc: (optional) When ``True``, datetime values are converted357 from local time to UTC time before being sent to the server. Datetimes received from358 the server are then converted back to local time. When ``False`` the client should use359 UTC date time values. Default is ``True``.360 :param str http_proxy: (optional) URL for a proxy server to use for all connections. The361 expected str format is ``[username:password@]111.222.333.444[:8080]``. Examples::362 192.168.0.1363 192.168.0.1:8888364 joe:user@192.168.0.1:8888365 :param bool connect: (optional) When ``True``, as soon as the :class:`~shotgun_api3.Shotgun`366 instance is created, a connection will be made to the Shotgun server to determine the367 server capabilities and confirm this version of the client is compatible with the server368 version. This is mostly used for testing. Default is ``True``.369 :param str ca_certs: (optional) path to an external SSL certificates file. By default, the370 Shotgun API will use its own built-in certificates file which stores root certificates371 for the most common Certificate Authorities (CAs). If you are using a corporate or372 internal CA, or are packaging an application into an executable, it may be necessary to373 point to your own certificates file. You can do this by passing in the full path to the374 file via this parameter or by setting the environment variable ``SHOTGUN_API_CACERTS``.375 In the case both are set, this parameter will take precedence.376 :param str login: The user login str to use to authenticate to the server when using user-based377 authentication. If provided, then ``password`` must be as well, and neither378 ``script_name`` nor ``api_key`` can be provided.379 .. seealso:: :ref:`authentication`380 :param str password: The password str to use to authenticate to the server when using user-based381 authentication. If provided, then ``login`` must be as well and neither ``script_name``382 nor ``api_key`` can be provided.383 See :ref:`authentication` for more info.384 :param str sudo_as_login: A user login string for the user whose permissions will be applied385 to all actions. Event log entries will be generated showing this user performing all386 actions with an additional extra meta-data parameter ``sudo_actual_user`` indicating the387 script or user that is actually authenticated.388 :param str session_token: The session token to use to authenticate to the server. This389 can be used as an alternative to authenticating with a script user or regular user.390 You can retrieve the session token by running the391 :meth:`~shotgun_api3.Shotgun.get_session_token()` method.392 .. todo: Add this info to the Authentication section of the docs393 :param str auth_token: The authentication token required to authenticate to a server with394 two-factor authentication turned on. If provided, then ``login`` and ``password`` must395 be provided as well, and neither ``script_name`` nor ``api_key`` can be provided.396 .. note:: These tokens can be short lived so a session is established right away if an397 ``auth_token`` is provided. A398 :class:`~shotgun_api3.MissingTwoFactorAuthenticationFault` will be raised if the399 ``auth_token`` is invalid.400 .. todo: Add this info to the Authentication section of the docs401 .. note:: A note about proxy connections: If you are using Python <= v2.6.2, HTTPS402 connections through a proxy server will not work due to a bug in the :mod:`urllib2`403 library (see http://bugs.python.org/issue1424152). This will affect upload and404 download-related methods in the Shotgun API (eg. :meth:`~shotgun_api3.Shotgun.upload`,405 :meth:`~shotgun_api3.Shotgun.upload_thumbnail`,406 :meth:`~shotgun_api3.Shotgun.upload_filmstrip_thumbnail`,407 :meth:`~shotgun_api3.Shotgun.download_attachment`. Normal CRUD methods for passing JSON408 data should still work fine. If you cannot upgrade your Python installation, you can see409 the patch merged into Python v2.6.3 (http://hg.python.org/cpython/rev/0f57b30a152f/) and410 try and hack it into your installation but YMMV. For older versions of Python there411 are other patches that were proposed in the bug report that may help you as well.412 """413 # verify authentication arguments414 if session_token is not None:415 if script_name is not None or api_key is not None:416 raise ValueError("cannot provide both session_token "417 "and script_name/api_key")418 if login is not None or password is not None:419 raise ValueError("cannot provide both session_token "420 "and login/password")421 if login is not None or password is not None:422 if script_name is not None or api_key is not None:423 raise ValueError("cannot provide both login/password "424 "and script_name/api_key")425 if login is None:426 raise ValueError("password provided without login")427 if password is None:428 raise ValueError("login provided without password")429 if script_name is not None or api_key is not None:430 if script_name is None:431 raise ValueError("api_key provided without script_name")432 if api_key is None:433 raise ValueError("script_name provided without api_key")434 if auth_token is not None:435 if login is None or password is None:436 raise ValueError("must provide a user login and password with an auth_token")437 if script_name is not None or api_key is not None:438 raise ValueError("cannot provide an auth_code with script_name/api_key")439 # Can't use 'all' with python 2.4440 if len([x for x in [session_token, script_name, api_key, login, password] if x]) == 0:441 if connect:442 raise ValueError("must provide login/password, session_token or script_name/api_key")443 self.config = _Config()444 self.config.api_key = api_key445 self.config.script_name = script_name446 self.config.user_login = login447 self.config.user_password = password448 self.config.auth_token = auth_token449 self.config.session_token = session_token450 self.config.sudo_as_login = sudo_as_login451 self.config.convert_datetimes_to_utc = convert_datetimes_to_utc452 self.config.no_ssl_validation = NO_SSL_VALIDATION453 self.config.raw_http_proxy = http_proxy454 self._connection = None455 if ca_certs is not None:456 self.__ca_certs = ca_certs457 else:458 self.__ca_certs = os.environ.get('SHOTGUN_API_CACERTS')459 self.base_url = (base_url or "").lower()460 self.config.scheme, self.config.server, api_base, _, _ = \461 urlparse.urlsplit(self.base_url)462 if self.config.scheme not in ("http", "https"):463 raise ValueError("base_url must use http or https got '%s'" %464 self.base_url)465 self.config.api_path = urlparse.urljoin(urlparse.urljoin(466 api_base or "/", self.config.api_ver + "/"), "json")467 # if the service contains user information strip it out468 # copied from the xmlrpclib which turned the user:password into469 # and auth header470 auth, self.config.server = urllib.splituser(urlparse.urlsplit(base_url).netloc)471 if auth:472 auth = base64.encodestring(urllib.unquote(auth))473 self.config.authorization = "Basic " + auth.strip()474 # foo:bar@123.456.789.012:3456475 if http_proxy:476 # check if we're using authentication. Start from the end since there might be477 # @ in the user's password.478 p = http_proxy.rsplit("@", 1)479 if len(p) > 1:480 self.config.proxy_user, self.config.proxy_pass = \481 p[0].split(":", 1)482 proxy_server = p[1]483 else:484 proxy_server = http_proxy485 proxy_netloc_list = proxy_server.split(":", 1)486 self.config.proxy_server = proxy_netloc_list[0]487 if len(proxy_netloc_list) > 1:488 try:489 self.config.proxy_port = int(proxy_netloc_list[1])490 except ValueError:491 raise ValueError("Invalid http_proxy address '%s'. Valid " \492 "format is '123.456.789.012' or '123.456.789.012:3456'"\493 ". If no port is specified, a default of %d will be "\494 "used." % (http_proxy, self.config.proxy_port))495 # now populate self.config.proxy_handler496 if self.config.proxy_user and self.config.proxy_pass:497 auth_string = "%s:%s@" % (self.config.proxy_user, self.config.proxy_pass)498 else:499 auth_string = ""500 proxy_addr = "http://%s%s:%d" % (auth_string, self.config.proxy_server, self.config.proxy_port)501 self.config.proxy_handler = urllib2.ProxyHandler({self.config.scheme : proxy_addr})502 if ensure_ascii:503 self._json_loads = self._json_loads_ascii504 self.client_caps = ClientCapabilities()505 # this relies on self.client_caps being set first506 self.reset_user_agent()507 self._server_caps = None508 # test to ensure the the server supports the json API509 # call to server will only be made once and will raise error510 if connect:511 self.server_caps512 # Check for api_max_entities_per_page in the server info and change the record per page value if it is supplied.513 self.config.records_per_page = self.server_info.get('api_max_entities_per_page') or self.config.records_per_page514 # When using auth_token in a 2FA scenario we need to switch to session-based515 # authentication because the auth token will no longer be valid after a first use.516 if self.config.auth_token is not None:517 self.config.session_token = self.get_session_token()518 self.config.user_login = None519 self.config.user_password = None520 self.config.auth_token = None521 # ========================================================================522 # API Functions523 @property524 def server_info(self):525 """526 Property containing server information.527 >>> sg.server_info528 {'full_version': [6, 3, 15, 0], 'version': [6, 3, 15], ...}529 .. note::530 Beyond ``full_version`` and ``version`` which differ by the inclusion of the bugfix number, you should expect531 these values to be unsupported and for internal use only.532 :returns: dict of server information from :class:`ServerCapabilities` object533 :rtype: dict534 """535 return self.server_caps.server_info536 @property537 def server_caps(self):538 """539 Property containing :class:`ServerCapabilities` object.540 >>> sg.server_caps541 <shotgun_api3.shotgun.ServerCapabilities object at 0x10120d350>542 :returns: :class:`ServerCapabilities` object that describe the server the client is543 connected to.544 :rtype: :class:`ServerCapabilities` object545 """546 if not self._server_caps or (547 self._server_caps.host != self.config.server):548 self._server_caps = ServerCapabilities(self.config.server,549 self.info())550 return self._server_caps551 def connect(self):552 """553 Connect client to the server if it is not already connected.554 .. note:: The client will automatically connect to the server on demand. You only need to555 call this function if you wish to confirm the client can connect.556 """557 self._get_connection()558 self.info()559 return560 def close(self):561 """562 Close the current connection to the server.563 If the client needs to connect again it will do so automatically.564 """565 self._close_connection()566 return567 def info(self):568 """569 Get API-related metadata from the Shotgun server.570 >>> sg.info()571 {'full_version': [6, 3, 15, 0], 'version': [6, 3, 15], ...}572 .. note::573 Beyond ``full_version`` and ``version`` which differ by the inclusion of the bugfix number, you should expect574 these values to be unsupported and for internal use only.575 :returns: dict of the server metadata.576 :rtype: dict577 """578 return self._call_rpc("info", None, include_auth_params=False)579 def find_one(self, entity_type, filters, fields=None, order=None,580 filter_operator=None, retired_only=False, include_archived_projects=True,581 additional_filter_presets=None):582 """583 Shortcut for :meth:`~shotgun_api3.Shotgun.find` with ``limit=1`` so it returns a single584 result.585 >>> sg.find_one("Asset", [["id", "is", 32]], ["id", "code", "sg_status_list"])586 {'code': 'Gopher', 'id': 32, 'sg_status_list': 'ip', 'type': 'Asset'}587 :param str entity_type: Shotgun entity type as a string to find.588 :param list filters: list of filters to apply to the query.589 .. seealso:: :ref:`filter_syntax`590 :param list fields: Optional list of fields to include in each entity record returned.591 Defaults to ``["id"]``.592 :param int order: Optional list of fields to order the results by. List has the format::593 [{'field_name':'foo', 'direction':'asc'}, {'field_name':'bar', 'direction':'desc'}]594 Defaults to sorting by ``id`` in ascending order.595 :param str filter_operator: Operator to apply to the filters. Supported values are ``"all"``596 and ``"any"``. These are just another way of defining if the query is an AND or OR597 query. Defaults to ``"all"``.598 :param bool retired_only: Optional boolean when ``True`` will return only entities that have599 been retried. Defaults to ``False`` which returns only entities which have not been600 retired. There is no option to return both retired and non-retired entities in the601 same query.602 :param bool include_archived_projects: Optional boolean flag to include entities whose projects603 have been archived. Defaults to ``True``.604 :param additional_filter_presets: Optional list of presets to further filter the result605 set, list has the form::606 [{"preset_name": <preset_name>, <optional_param1>: <optional_value1>, ... }]607 Note that these filters are ANDed together and ANDed with the 'filter'608 argument.609 For details on supported presets and the format of this parameter see610 :ref:`additional_filter_presets`611 :returns: Dictionary representing a single matching entity with the requested fields,612 and the defaults ``"id"`` and ``"type"`` which are always included.613 :rtype: dict614 """615 results = self.find(entity_type, filters, fields, order,616 filter_operator, 1, retired_only, include_archived_projects=include_archived_projects,617 additional_filter_presets=additional_filter_presets)618 if results:619 return results[0]620 return None621 def find(self, entity_type, filters, fields=None, order=None,622 filter_operator=None, limit=0, retired_only=False, page=0,623 include_archived_projects=True, additional_filter_presets=None):624 """625 Find entities matching the given filters.626 >>> # Find Character Assets in Sequence 100_FOO627 >>> # -------------628 >>> fields = ['id', 'code', 'sg_asset_type']629 >>> sequence_id = 2 # Sequence "100_FOO"630 >>> project_id = 4 # Demo Project631 >>> filters = [632 ... ['project', 'is', {'type': 'Project', 'id': project_id}],633 ... ['sg_asset_type', 'is', 'Character'],634 ... ['sequences', 'is', {'type': 'Sequence', 'id': sequence_id}]635 ... ]636 >>> assets= sg.find("Asset",filters,fields)637 [{'code': 'Gopher', 'id': 32, 'sg_asset_type': 'Character', 'type': 'Asset'},638 {'code': 'Cow', 'id': 33, 'sg_asset_type': 'Character', 'type': 'Asset'},639 {'code': 'Bird_1', 'id': 35, 'sg_asset_type': 'Character', 'type': 'Asset'},640 {'code': 'Bird_2', 'id': 36, 'sg_asset_type': 'Character', 'type': 'Asset'},641 {'code': 'Bird_3', 'id': 37, 'sg_asset_type': 'Character', 'type': 'Asset'},642 {'code': 'Raccoon', 'id': 45, 'sg_asset_type': 'Character', 'type': 'Asset'},643 {'code': 'Wet Gopher', 'id': 149, 'sg_asset_type': 'Character', 'type': 'Asset'}]644 You can drill through single entity links to filter on fields or display linked fields.645 This is often called "deep linking" or using "dot syntax".646 .. seealso:: :ref:`filter_syntax`647 >>> # Find Versions created by Tasks in the Animation Pipeline Step648 >>> # -------------649 >>> fields = ['id', 'code']650 >>> pipeline_step_id = 2 # Animation Step ID651 >>> project_id = 4 # Demo Project652 >>> # you can drill through single-entity link fields653 >>> filters = [654 ... ['project','is', {'type': 'Project','id': project_id}],655 ... ['sg_task.Task.step.Step.id', 'is', pipeline_step_id]656 >>> ]657 >>> sg.find("Version", filters, fields)658 [{'code': 'scene_010_anim_v001', 'id': 42, 'type': 'Version'},659 {'code': 'scene_010_anim_v002', 'id': 134, 'type': 'Version'},660 {'code': 'bird_v001', 'id': 137, 'type': 'Version'},661 {'code': 'birdAltBlue_v002', 'id': 236, 'type': 'Version'}]662 :param str entity_type: Shotgun entity type to find.663 :param list filters: list of filters to apply to the query.664 .. seealso:: :ref:`filter_syntax`665 :param list fields: Optional list of fields to include in each entity record returned.666 Defaults to ``["id"]``.667 :param list order: Optional list of dictionaries defining how to order the results of the668 query. Each dictionary contains the ``field_name`` to order by and the ``direction``669 to sort::670 [{'field_name':'foo', 'direction':'asc'}, {'field_name':'bar', 'direction':'desc'}]671 Defaults to sorting by ``id`` in ascending order.672 :param str filter_operator: Operator to apply to the filters. Supported values are ``"all"``673 and ``"any"``. These are just another way of defining if the query is an AND or OR674 query. Defaults to ``"all"``.675 :param int limit: Optional limit to the number of entities to return. Defaults to ``0`` which676 returns all entities that match.677 :param int page: Optional page of results to return. Use this together with the ``limit``678 parameter to control how your query results are paged. Defaults to ``0`` which returns679 all entities that match.680 :param bool retired_only: Optional boolean when ``True`` will return only entities that have681 been retried. Defaults to ``False`` which returns only entities which have not been682 retired. There is no option to return both retired and non-retired entities in the683 same query.684 :param bool include_archived_projects: Optional boolean flag to include entities whose projects685 have been archived. Defaults to ``True``.686 :param additional_filter_presets: Optional list of presets to further filter the result687 set, list has the form::688 [{"preset_name": <preset_name>, <optional_param1>: <optional_value1>, ... }]689 Note that these filters are ANDed together and ANDed with the 'filter'690 argument.691 For details on supported presets and the format of this parameter see692 :ref:`additional_filter_presets`693 :returns: list of dictionaries representing each entity with the requested fields, and the694 defaults ``"id"`` and ``"type"`` which are always included.695 :rtype: list696 """697 if not isinstance(limit, int) or limit < 0:698 raise ValueError("limit parameter must be a positive integer")699 if not isinstance(page, int) or page < 0:700 raise ValueError("page parameter must be a positive integer")701 if isinstance(filters, (list, tuple)):702 filters = _translate_filters(filters, filter_operator)703 elif filter_operator:704 # TODO: Not sure if this test is correct, replicated from prev api705 raise ShotgunError("Deprecated: Use of filter_operator for find()"706 " is not valid any more. See the documentation on find()")707 if not include_archived_projects:708 # This defaults to True on the server (no argument is sent)709 # So we only need to check the server version if it is False710 self.server_caps.ensure_include_archived_projects()711 if additional_filter_presets:712 self.server_caps.ensure_support_for_additional_filter_presets()713 params = self._construct_read_parameters(entity_type,714 fields,715 filters,716 retired_only,717 order,718 include_archived_projects,719 additional_filter_presets)720 if self.server_caps.ensure_return_image_urls_support():721 params['api_return_image_urls'] = True722 if self.server_caps.ensure_paging_info_without_counts_support():723 paging_info_param = "return_paging_info_without_counts"724 else:725 paging_info_param = "return_paging_info"726 params[paging_info_param] = False727 if limit and limit <= self.config.records_per_page:728 params["paging"]["entities_per_page"] = limit729 # If page isn't set and the limit doesn't require pagination,730 # then trigger the faster code path.731 if page == 0:732 page = 1733 # if page is specified, then only return the page of records requested734 if page != 0:735 params["paging"]["current_page"] = page736 records = self._call_rpc("read", params).get("entities", [])737 return self._parse_records(records)738 params[paging_info_param] = True739 records = []740 if self.server_caps.ensure_paging_info_without_counts_support():741 has_next_page = True742 while has_next_page:743 result = self._call_rpc("read", params)744 records.extend(result.get("entities"))745 if limit and len(records) >= limit:746 records = records[:limit]747 break748 has_next_page = result["paging_info"]["has_next_page"]749 params['paging']['current_page'] += 1750 else:751 result = self._call_rpc("read", params)752 while result.get("entities"):753 records.extend(result.get("entities"))754 if limit and len(records) >= limit:755 records = records[:limit]756 break757 if len(records) == result["paging_info"]["entity_count"]:758 break759 params['paging']['current_page'] += 1760 result = self._call_rpc("read", params)761 return self._parse_records(records)762 def _construct_read_parameters(self,763 entity_type,764 fields,765 filters,766 retired_only,767 order,768 include_archived_projects,769 additional_filter_presets):770 params = {}771 params["type"] = entity_type772 params["return_fields"] = fields or ["id"]773 params["filters"] = filters774 params["return_only"] = (retired_only and 'retired') or "active"775 params["paging"] = { "entities_per_page": self.config.records_per_page,776 "current_page": 1 }777 if additional_filter_presets:778 params["additional_filter_presets"] = additional_filter_presets;779 if include_archived_projects is False:780 # Defaults to True on the server, so only pass it if it's False781 params["include_archived_projects"] = False782 if order:783 sort_list = []784 for sort in order:785 if sort.has_key('column'):786 # TODO: warn about deprecation of 'column' param name787 sort['field_name'] = sort['column']788 sort.setdefault("direction", "asc")789 sort_list.append({790 'field_name': sort['field_name'],791 'direction' : sort['direction']792 })793 params['sorts'] = sort_list794 return params795 def _add_project_param(self, params, project_entity):796 if project_entity and self.server_caps.ensure_per_project_customization():797 params["project"] = project_entity798 return params799 def summarize(self,800 entity_type,801 filters,802 summary_fields,803 filter_operator=None,804 grouping=None,805 include_archived_projects=True):806 """807 Summarize field data returned by a query.808 This provides the same functionality as the summaries in the UI. You can specify one or809 more fields to summarize, choose the summary type for each, and optionally group the810 results which will return summary information for each group as well as the total for811 the query.812 **Example: Count all Assets for a Project**813 >>> sg.summarize(entity_type='Asset',814 ... filters = [['project', 'is', {'type':'Project', 'id':4}]],815 ... summary_fields=[{'field':'id', 'type':'count'}])816 {'groups': [], 'summaries': {'id': 15}}817 ``summaries`` contains the total summary for the query. Each key is the field summarized818 and the value is the result of the summary operation for the entire result set.819 .. note::820 You cannot perform more than one summary on a field at a time, but you can summarize821 several different fields in the same call.822 **Example: Count all Assets for a Project, grouped by sg_asset_type**823 >>> sg.summarize(entity_type='Asset',824 ... filters=[['project', 'is', {'type': 'Project', 'id': 4}]],825 ... summary_fields=[{'field': 'id', 'type': 'count'}],826 ... grouping=[{'field': 'sg_asset_type', 'type': 'exact', 'direction': 'asc'}])827 {'groups': [{'group_name': 'Character','group_value': 'Character', 'summaries': {'id': 3}},828 {'group_name': 'Environment','group_value': 'Environment', 'summaries': {'id': 3}},829 {'group_name': 'Matte Painting', 'group_value': 'Matte Painting', 'summaries': {'id': 1}},830 {'group_name': 'Prop', 'group_value': 'Prop', 'summaries': {'id': 4}},831 {'group_name': 'Vehicle', 'group_value': 'Vehicle', 'summaries': {'id': 4}}],832 'summaries': {'id': 15}}833 - ``summaries`` contains the total summary for the query.834 - ``groups`` contains the summary for each group.835 - ``group_name`` is the display name for the group.836 - ``group_value`` is the actual value of the grouping value. This is often the same as837 ``group_name`` but in the case when grouping by entity, the ``group_name`` may be838 ``PuppyA`` and the group_value would be839 ``{'type':'Asset','id':922,'name':'PuppyA'}``.840 - ``summaries`` contains the summary calculation dict for each field requested.841 **Example: Count all Tasks for a Sequence and find the latest due_date**842 >>> sg.summarize(entity_type='Task',843 ... filters = [844 ... ['entity.Shot.sg_sequence', 'is', {'type':'Sequence', 'id':2}],845 ... ['sg_status_list', 'is_not', 'na']],846 ... summary_fields=[{'field':'id', 'type':'count'},847 ... {'field':'due_date','type':'latest'}])848 {'groups': [], 'summaries': {'due_date': '2013-07-05', 'id': 30}}849 This shows that the there are 30 Tasks for Shots in the Sequence and the latest ``due_date``850 of any Task is ``2013-07-05``.851 **Example: Count all Tasks for a Sequence, find the latest due_date and group by Shot**852 >>> sg.summarize(entity_type='Task',853 ... filters = [854 ... ['entity.Shot.sg_sequence', 'is', {'type': 'Sequence', 'id': 2}],855 ... ['sg_status_list', 'is_not', 'na']],856 ... summary_fields=[{'field': 'id', 'type': 'count'}, {'field': 'due_date', 'type': 'latest'}],857 ... grouping=[{'field': 'entity', 'type': 'exact', 'direction': 'asc'}]))858 {'groups': [{'group_name': 'shot_010',859 'group_value': {'id': 2, 'name': 'shot_010', 'type': 'Shot', 'valid': 'valid'},860 'summaries': {'due_date': '2013-06-18', 'id': 10}},861 {'group_name': 'shot_020',862 'group_value': {'id': 3, 'name': 'shot_020', 'type': 'Shot', 'valid': 'valid'},863 'summaries': {'due_date': '2013-06-28', 'id': 10}},864 {'group_name': 'shot_030',865 'group_value': {'id': 4, 'name': 'shot_030', 'type': 'Shot', 'valid': 'valid'},866 'summaries': {'due_date': '2013-07-05', 'id': 10}}],867 'summaries': {'due_date': '2013-07-05', 'id': 30}}868 This shows that the there are 30 Tasks for Shots in the Sequence and the latest ``due_date``869 of any Task is ``2013-07-05``. Because the summary is grouped by ``entity``, we can also870 see the summaries for each Shot returned. Each Shot has 10 Tasks and the latest ``due_date``871 for each Shot. The difference between ``group_name`` and ``group_value`` is highlighted in872 this example as the name of the Shot is different from its value.873 **Example: Count all Tasks for a Sequence, find the latest due_date, group by Shot and874 Pipeline Step**875 >>> sg.summarize(entity_type='Task',876 ... filters = [877 ... ['entity.Shot.sg_sequence', 'is', {'type': 'Sequence', 'id': 2}],878 ... ['sg_status_list', 'is_not', 'na']],879 ... summary_fields=[{'field': 'id', 'type': 'count'},880 ... {'field': 'due_date', 'type': 'latest'}],881 ... grouping=[{'field': 'entity', 'type': 'exact', 'direction': 'asc'},882 ... {'field': 'step', 'type': 'exact', 'direction': 'asc'}])883 {'groups': [{'group_name': 'shot_010',884 'group_value': {'id': 2, 'name': 'shot_010', 'type': 'Shot', 'valid': 'valid'},885 'groups': [{'group_name': 'Client',886 'group_value': {'id': 1, 'name': 'Client', 'type': 'Step', 'valid': 'valid'},887 'summaries': {'due_date': '2013-05-04', 'id': 1}},888 {'group_name': 'Online',889 'group_value': {'id': 2, 'name': 'Online', 'type': 'Step', 'valid': 'valid'},890 'summaries': {'due_date': '2013-05-05', 'id': 1}},891 ...892 ... truncated for brevity893 ...894 {'group_name': 'Comp',895 'group_value': {'id': 8, 'name': 'Comp', 'type': 'Step', 'valid': 'valid'},896 'summaries': {'due_date': '2013-06-18', 'id': 1}}],897 'summaries': {'due_date': '2013-06-18', 'id': 10}},898 {'group_name': 'shot_020',899 'group_value': {'id': 3, 'name': 'shot_020', 'type': 'Shot', 'valid': 'valid'},900 'groups': [{'group_name': 'Client',901 'group_value': {'id': 1, 'name': 'Client', 'type': 'Step', 'valid': 'valid'},902 'summaries': {'due_date': '2013-05-15', 'id': 1}},903 {'group_name': 'Online',904 'group_value': {'id': 2, 'name': 'Online', 'type': 'Step', 'valid': 'valid'},905 'summaries': {'due_date': '2013-05-16', 'id': 1}},906 ...907 ... truncated for brevity908 ...909 {'group_name': 'Comp',910 'group_value': {'id': 8, 'name': 'Comp', 'type': 'Step', 'valid': 'valid'},911 'summaries': {'due_date': '2013-06-28', 'id': 1}}],912 'summaries': {'due_date': '2013-06-28', 'id': 10}},913 {'group_name': 'shot_030',914 'group_value': {'id': 4, 'name': 'shot_030', 'type': 'Shot', 'valid': 'valid'},915 'groups': [{'group_name': 'Client',916 'group_value': {'id': 1, 'name': 'Client', 'type': 'Step', 'valid': 'valid'},917 'summaries': {'due_date': '2013-05-20', 'id': 1}},918 {'group_name': 'Online',919 'group_value': {'id': 2, 'name': 'Online', 'type': 'Step', 'valid': 'valid'},920 'summaries': {'due_date': '2013-05-21', 'id': 1}},921 ...922 ... truncated for brevity923 ...924 {'group_name': 'Comp',925 'group_value': {'id': 8, 'name': 'Comp', 'type': 'Step', 'valid': 'valid'},926 'summaries': {'due_date': '2013-07-05', 'id': 1}}],927 'summaries': {'due_date': '2013-07-05', 'id': 10}}],928 'summaries': {'due_date': '2013-07-05', 'id': 30}}929 When grouping my more than one field, the grouping structure is repeated for each sub-group930 and summary values are returned for each group on each level.931 :param str entity_type: The entity type to summarize932 :param list filters: A list of conditions used to filter the find query. Uses the same933 syntax as :meth:`~shotgun_api3.Shotgun.find` method.934 :param list summary_fields: A list of dictionaries with the following keys:935 :field: The internal Shotgun field name you are summarizing.936 :type: The type of summary you are performing on the field. Summary types can be any of937 ``record_count``, ``count``, ``sum``, ``maximum``, ``minimum``, ``average``,938 ``earliest``, ``latest``, ``percentage``, ``status_percentage``, ``status_list``,939 ``checked``, ``unchecked`` depending on the type of field you're summarizing.940 :param str filter_operator: Operator to apply to the filters. Supported values are ``"all"``941 and ``"any"``. These are just another way of defining if the query is an AND or OR942 query. Defaults to ``"all"``.943 :param list grouping: Optional list of dicts with the following keys:944 :field: a string indicating the internal Shotgun field name on ``entity_type`` to945 group results by.946 :type: A string indicating the type of grouping to perform for each group.947 Valid types depend on the type of field you are grouping on and can be one of948 ``exact``, ``tens``, ``hundreds``, ``thousands``, ``tensofthousands``,949 ``hundredsofthousands``, ``millions``, ``day``, ``week``, ``month``,950 ``quarter``,``year``, ``clustered_date``, ``oneday``, ``fivedays``,951 ``entitytype``, ``firstletter``.952 :direction: A string that sets the order to display the grouped results. Valid953 options are ``asc`` and ``desc``. Defaults to ``asc``.954 :returns: dictionary containing grouping and summaries keys.955 :rtype: dict956 """957 if not isinstance(grouping, list) and grouping is not None:958 msg = "summarize() 'grouping' parameter must be a list or None"959 raise ValueError(msg)960 if isinstance(filters, (list, tuple)):961 filters = _translate_filters(filters, filter_operator)962 if not include_archived_projects:963 # This defaults to True on the server (no argument is sent)964 # So we only need to check the server version if it is False965 self.server_caps.ensure_include_archived_projects()966 params = {"type": entity_type,967 "summaries": summary_fields,968 "filters": filters}969 if include_archived_projects is False:970 # Defaults to True on the server, so only pass it if it's False971 params["include_archived_projects"] = False972 if grouping is not None:973 params['grouping'] = grouping974 records = self._call_rpc('summarize', params)975 return records976 def create(self, entity_type, data, return_fields=None):977 """978 Create a new entity of the specified ``entity_type``.979 >>> data = {980 ... "project": {"type": "Project", "id": 161},981 ... "sg_sequence": {"type": "Sequence", "id": 109},982 ... "code": "001_100",983 ... 'sg_status_list': "ip"984 ... }985 >>> sg.create('Shot', data)986 {'code': '001_100',987 'id': 2557,988 'project': {'id': 161, 'name': 'Pied Piper', 'type': 'Project'},989 'sg_sequence': {'id': 109, 'name': 'Sequence 001', 'type': 'Sequence'},990 'sg_status_list': 'ip',991 'type': 'Shot'}992 :param str entity_type: Shotgun entity type to create.993 :param dict data: Dictionary of fields and corresponding values to set on the new entity. If994 ``image`` or ``filmstrip_image`` fields are provided, the file path will be uploaded995 to the server automatically.996 :param list return_fields: Optional list of additional field values to return from the new997 entity. Defaults to ``id`` field.998 :returns: Shotgun entity dictionary containing the field/value pairs of all of the fields999 set from the ``data`` parameter as well as the defaults ``type`` and ``id``. If any1000 additional fields were provided using the ``return_fields`` parameter, these would be1001 included as well.1002 :rtype: dict1003 """1004 data = data.copy()1005 if not return_fields:1006 return_fields = ["id"]1007 upload_image = None1008 if 'image' in data:1009 upload_image = data.pop('image')1010 upload_filmstrip_image = None1011 if 'filmstrip_image' in data:1012 if not self.server_caps.version or self.server_caps.version < (3, 1, 0):1013 raise ShotgunError("Filmstrip thumbnail support requires server version 3.1 or "\1014 "higher, server is %s" % (self.server_caps.version,))1015 upload_filmstrip_image = data.pop('filmstrip_image')1016 params = {1017 "type" : entity_type,1018 "fields" : self._dict_to_list(data),1019 "return_fields" : return_fields1020 }1021 record = self._call_rpc("create", params, first=True)1022 result = self._parse_records(record)[0]1023 if upload_image:1024 image_id = self.upload_thumbnail(entity_type, result['id'],1025 upload_image)1026 image = self.find_one(entity_type, [['id', 'is', result.get('id')]],1027 fields=['image'])1028 result['image'] = image.get('image')1029 if upload_filmstrip_image:1030 filmstrip_id = self.upload_filmstrip_thumbnail(entity_type, result['id'], upload_filmstrip_image)1031 filmstrip = self.find_one(entity_type,1032 [['id', 'is', result.get('id')]],1033 fields=['filmstrip_image'])1034 result['filmstrip_image'] = filmstrip.get('filmstrip_image')1035 return result1036 def update(self, entity_type, entity_id, data, multi_entity_update_modes=None):1037 """1038 Update the specified entity with the supplied data.1039 >>> shots = [1040 ... {'type':'Shot', 'id':'40435'},1041 ... {'type':'Shot', 'id':'40438'},1042 ... {'type':'Shot', 'id':'40441'}]1043 >>> data = {1044 ... 'shots': shots_asset_is_in,1045 ... 'sg_status_list':'rev'}1046 >>> sg.update("Asset", 55, data)1047 {'type': 'Shot',1048 'id': 55,1049 'sg_status_`list`': 'rev',1050 'shots': [{'id': 40435, 'name': '100_010', 'type': 'Shot', 'valid': 'valid'},1051 {'id': 40438, 'name': '100_040', 'type': 'Shot', 'valid': 'valid'},1052 {'id': 40441, 'name': '100_070', 'type': 'Shot', 'valid': 'valid'}]1053 }1054 :param str entity_type: Entity type to update.1055 :param id entity_id: id of the entity to update.1056 :param dict data: key/value pairs where key is the field name and value is the value to set1057 for that field. This method does not restrict the updating of fields hidden in the web1058 UI via the Project Tracking Settings panel.1059 :param dict multi_entity_update_modes: Optional dict indicating what update mode to use1060 when updating a multi-entity link field. The keys in the dict are the fields to set1061 the mode for, and the values from the dict are one of ``set``, ``add``, or ``remove``.1062 Defaults to ``set``.1063 ::1064 multi_entity_update_modes={"shots": "add", "assets": "remove"}1065 :returns: Dictionary of the fields updated, with the default keys `type` and `id` added as well.1066 :rtype: dict1067 """1068 data = data.copy()1069 upload_image = None1070 if 'image' in data and data['image'] is not None:1071 upload_image = data.pop('image')1072 upload_filmstrip_image = None1073 if 'filmstrip_image' in data:1074 if not self.server_caps.version or self.server_caps.version < (3, 1, 0):1075 raise ShotgunError("Filmstrip thumbnail support requires server version 3.1 or "\1076 "higher, server is %s" % (self.server_caps.version,))1077 upload_filmstrip_image = data.pop('filmstrip_image')1078 if data:1079 params = {1080 "type" : entity_type,1081 "id" : entity_id,1082 "fields" : self._dict_to_list(1083 data,1084 extra_data=self._dict_to_extra_data(1085 multi_entity_update_modes, "multi_entity_update_mode"))1086 }1087 record = self._call_rpc("update", params)1088 result = self._parse_records(record)[0]1089 else:1090 result = {'id': entity_id, 'type': entity_type}1091 if upload_image:1092 image_id = self.upload_thumbnail(entity_type, entity_id,1093 upload_image)1094 image = self.find_one(entity_type, [['id', 'is', result.get('id')]],1095 fields=['image'])1096 result['image'] = image.get('image')1097 if upload_filmstrip_image:1098 filmstrip_id = self.upload_filmstrip_thumbnail(entity_type, result['id'], upload_filmstrip_image)1099 filmstrip = self.find_one(entity_type,1100 [['id', 'is', result.get('id')]],1101 fields=['filmstrip_image'])1102 result['filmstrip_image'] = filmstrip.get('filmstrip_image')1103 return result1104 def delete(self, entity_type, entity_id):1105 """1106 Retire the specified entity.1107 Entities in Shotgun are not "deleted" destructively, they are instead, "retired". This1108 means they are placed in the trash where they are no longer accessible to users.1109 The entity can be brought back to life using :meth:`~shotgun_api3.Shotgun.revive`.1110 >>> sg.delete("Shot", 2557)1111 True1112 :param str entity_type: Shotgun entity type to delete.1113 :param id entity_id: ``id`` of the entity to delete.1114 :returns: ``True`` if the entity was deleted, ``False`` otherwise (for example, if the1115 entity was already deleted).1116 :rtype: bool1117 :raises: :class:`Fault` if entity does not exist (deleted or not).1118 """1119 params = {1120 "type" : entity_type,1121 "id" : entity_id1122 }1123 return self._call_rpc("delete", params)1124 def revive(self, entity_type, entity_id):1125 """1126 Revive an entity that has previously been deleted.1127 >>> sg.revive("Shot", 860)1128 True1129 :param str entity_type: Shotgun entity type to revive.1130 :param int entity_id: id of the entity to revive.1131 :returns: ``True`` if the entity was revived, ``False`` otherwise (e.g. if the1132 entity is not currently retired).1133 :rtype: bool1134 """1135 params = {1136 "type" : entity_type,1137 "id" : entity_id1138 }1139 return self._call_rpc("revive", params)1140 def batch(self, requests):1141 """1142 Make a batch request of several :meth:`~shotgun_api3.Shotgun.create`,1143 :meth:`~shotgun_api3.Shotgun.update`, and :meth:`~shotgun_api3.Shotgun.delete` calls.1144 All requests are performed within a transaction, so either all will complete or none will.1145 Ex. Make a bunch of shots::1146 batch_data = []1147 for i in range(1,100):1148 data = {1149 "code": "shot_%04d" % i,1150 "project": project1151 }1152 batch_data.append({"request_type": "create", "entity_type": "Shot", "data": data})1153 sg.batch(batch_data)1154 Example output::1155 [{'code': 'shot_0001',1156 'type': 'Shot',1157 'id': 3624,1158 'project': {'id': 4, 'name': 'Demo Project', 'type': 'Project'}},1159 ...1160 ... and a bunch more ...1161 ...1162 {'code': 'shot_0099',1163 'type': 'Shot',1164 'id': 3722,1165 'project': {'id': 4, 'name': 'Demo Project', 'type': 'Project'}}]1166 Ex. All three types of requests in one batch::1167 batch_data = [1168 {"request_type": "create", "entity_type": "Shot", "data": {"code": "New Shot 1", "project": project}},1169 {"request_type": "update", "entity_type": "Shot", "entity_id": 3624, "data": {"code": "Changed 1"}},1170 {"request_type": "delete", "entity_type": "Shot", "entity_id": 3624}1171 ]1172 sg.batch(batch_data)1173 Example output::1174 [{'code': 'New Shot 1', 'type': 'Shot', 'id': 3723, 'project': {'id': 4, 'name': 'Demo Project', 'type': 'Project'}},1175 {'code': 'Changed 1', 'type': 'Shot', 'id': 3624},1176 True]1177 :param list requests: A list of dict's of the form which have a request_type key and also1178 specifies:1179 - create: ``entity_type``, data dict of fields to set1180 - update: ``entity_type``, ``entity_id``, data dict of fields to set, and optionally ``multi_entity_update_modes``1181 - delete: ``entity_type`` and entity_id1182 :returns: A list of values for each operation. Create and update requests return a dict of1183 the fields updated. Delete requests return ``True`` if the entity was deleted.1184 :rtype: list1185 """1186 if not isinstance(requests, list):1187 raise ShotgunError("batch() expects a list. Instead was sent "\1188 "a %s" % type(requests))1189 # If we have no requests, just return an empty list immediately.1190 # Nothing to process means nothing to get results of.1191 if len(requests) == 0:1192 return []1193 calls = []1194 def _required_keys(message, required_keys, data):1195 missing = set(required_keys) - set(data.keys())1196 if missing:1197 raise ShotgunError("%s missing required key: %s. "\1198 "Value was: %s." % (message, ", ".join(missing), data))1199 for req in requests:1200 _required_keys("Batched request",1201 ['request_type', 'entity_type'],1202 req)1203 request_params = {'request_type': req['request_type'],1204 "type" : req["entity_type"]}1205 if req["request_type"] == "create":1206 _required_keys("Batched create request", ['data'], req)1207 request_params['fields'] = self._dict_to_list(req["data"])1208 request_params["return_fields"] = req.get("return_fields") or["id"]1209 elif req["request_type"] == "update":1210 _required_keys("Batched update request",1211 ['entity_id', 'data'],1212 req)1213 request_params['id'] = req['entity_id']1214 request_params['fields'] = self._dict_to_list(req["data"],1215 extra_data=self._dict_to_extra_data(1216 req.get("multi_entity_update_modes"),1217 "multi_entity_update_mode"))1218 if "multi_entity_update_mode" in req:1219 request_params['multi_entity_update_mode'] = req["multi_entity_update_mode"]1220 elif req["request_type"] == "delete":1221 _required_keys("Batched delete request", ['entity_id'], req)1222 request_params['id'] = req['entity_id']1223 else:1224 raise ShotgunError("Invalid request_type '%s' for batch" % (1225 req["request_type"]))1226 calls.append(request_params)1227 records = self._call_rpc("batch", calls)1228 return self._parse_records(records)1229 def work_schedule_read(self, start_date, end_date, project=None, user=None):1230 """1231 Return the work day rules for a given date range.1232 .. versionadded:: 3.0.91233 Requires Shotgun server v3.2.0+1234 This returns the defined WorkDayRules between the ``start_date`` and ``end_date`` inclusive1235 as a dict where the key is the date and the value is another dict describing the rule for1236 that date.1237 Rules are represented by a dict with the following keys:1238 :description: the description entered into the work day rule exception if applicable.1239 :reason: one of six options:1240 - STUDIO_WORK_WEEK: standard studio schedule applies1241 - STUDIO_EXCEPTION: studio-wide exception applies1242 - PROJECT_WORK_WEEK: standard project schedule applies1243 - PROJECT_EXCEPTION: project-specific exception applies1244 - USER_WORK_WEEK: standard user work week applies1245 - USER_EXCEPTION: user-specific exception applies1246 :working: boolean indicating whether it is a "working" day or not.1247 >>> sg.work_schedule_read("2015-12-21", "2015-12-25")1248 {'2015-12-21': {'description': None,1249 'reason': 'STUDIO_WORK_WEEK',1250 'working': True},1251 '2015-12-22': {'description': None,1252 'reason': 'STUDIO_WORK_WEEK',1253 'working': True},1254 '2015-12-23': {'description': None,1255 'reason': 'STUDIO_WORK_WEEK',1256 'working': True},1257 '2015-12-24': {'description': 'Closed for Christmas Eve',1258 'reason': 'STUDIO_EXCEPTION',1259 'working': False},1260 '2015-12-25': {'description': 'Closed for Christmas',1261 'reason': 'STUDIO_EXCEPTION',1262 'working': False}}1263 :param str start_date: Start date of date range. ``YYYY-MM-DD``1264 :param str end_date: End date of date range. ``YYYY-MM-DD``1265 :param dict project: Optional Project entity to query `WorkDayRules` for.1266 :param dict user: Optional HumanUser entity to query WorkDayRules for.1267 :returns: Complex dict containing each date and the WorkDayRule defined for that date1268 between the ``start_date`` and ``end date`` inclusive. See above for details.1269 :rtype: dict1270 """1271 if not self.server_caps.version or self.server_caps.version < (3, 2, 0):1272 raise ShotgunError("Work schedule support requires server version 3.2 or "\1273 "higher, server is %s" % (self.server_caps.version,))1274 if not isinstance(start_date, str) or not isinstance(end_date, str):1275 raise ShotgunError("The start_date and end_date arguments must be strings in YYYY-MM-DD format")1276 params = dict(1277 start_date=start_date,1278 end_date=end_date,1279 project=project,1280 user=user1281 )1282 return self._call_rpc('work_schedule_read', params)1283 def work_schedule_update(self, date, working, description=None, project=None, user=None,1284 recalculate_field=None):1285 """1286 Update the work schedule for a given date.1287 .. versionadded:: 3.0.91288 Requires Shotgun server v3.2.0+1289 If neither ``project`` nor ``user`` are passed in, the studio work schedule will be updated.1290 ``project`` and ``user`` can only be used exclusively of each other.1291 >>> sg.work_schedule_update ("2015-12-31", working=False,1292 ... description="Studio closed for New Years Eve", project=None,1293 ... user=None, recalculate_field=None)1294 {'date': '2015-12-31',1295 'description': "Studio closed for New Years Eve",1296 'project': None,1297 'user': None,1298 'working': False}1299 :param str date: Date of WorkDayRule to update. ``YYY-MM-DD``1300 :param bool working: Indicates whether the day is a working day or not.1301 :param str description: Optional reason for time off.1302 :param dict project: Optional Project entity to assign the rule to. Cannot be used with the1303 ``user`` param.1304 :param dict user: Optional HumanUser entity to assign the rule to. Cannot be used with the1305 ``project`` param.1306 :param str recalculate_field: Optional schedule field that will be recalculated on Tasks1307 when they are affected by a change in working schedule. Options are ``due_date`` or1308 ``duration``. Defaults to the value set in the Shotgun web application's Site1309 Preferences.1310 :returns: dict containing key/value pairs for each value of the work day rule updated.1311 :rtype: dict1312 """1313 if not self.server_caps.version or self.server_caps.version < (3, 2, 0):1314 raise ShotgunError("Work schedule support requires server version 3.2 or "\1315 "higher, server is %s" % (self.server_caps.version,))1316 if not isinstance(date, str):1317 raise ShotgunError("The date argument must be string in YYYY-MM-DD format")1318 params = dict(1319 date=date,1320 working=working,1321 description=description,1322 project=project,1323 user=user,1324 recalculate_field=recalculate_field1325 )1326 return self._call_rpc('work_schedule_update', params)1327 def follow(self, user, entity):1328 """1329 Add the entity to the user's followed entities.1330 If the user is already following the entity, the method will succeed but nothing will be1331 changed on the server-side.1332 >>> sg.follow({"type": "HumanUser", "id": 42}, {"type": "Shot", "id": 2050})1333 {'followed': True, 'user': {'type': 'HumanUser', 'id': 42},1334 'entity': {'type': 'Shot', 'id': 2050}}1335 :param dict user: User entity that will follow the entity.1336 :param dict entity: Shotgun entity to be followed.1337 :returns: dict with ``"followed": True`` as well as key/values for the params that were1338 passed in.1339 :rtype: dict1340 """1341 if not self.server_caps.version or self.server_caps.version < (5, 1, 22):1342 raise ShotgunError("Follow support requires server version 5.2 or "\1343 "higher, server is %s" % (self.server_caps.version,))1344 params = dict(1345 user=user,1346 entity=entity1347 )1348 return self._call_rpc('follow', params)1349 def unfollow(self, user, entity):1350 """1351 Remove entity from the user's followed entities.1352 This does nothing if the user is not following the entity.1353 >>> sg.unfollow({"type": "HumanUser", "id": 42}, {"type": "Shot", "id": 2050})1354 {'entity': {'type': 'Shot', 'id': 2050}, 'user': {'type': 'HumanUser', 'id': 42},1355 'unfollowed': True}1356 :param dict user: User entity that will unfollow the entity.1357 :param dict entity: Entity to be unfollowed1358 :returns: dict with ``"unfollowed": True`` as well as key/values for the params that were1359 passed in.1360 :rtype: dict1361 """1362 if not self.server_caps.version or self.server_caps.version < (5, 1, 22):1363 raise ShotgunError("Follow support requires server version 5.2 or "\1364 "higher, server is %s" % (self.server_caps.version,))1365 params = dict(1366 user=user,1367 entity=entity1368 )1369 return self._call_rpc('unfollow', params)1370 def followers(self, entity):1371 """1372 Return all followers for an entity.1373 >>> sg.followers({"type": "Shot", "id": 2050})1374 [{'status': 'act', 'valid': 'valid', 'type': 'HumanUser', 'name': 'Richard Hendriks',1375 'id': 42},1376 {'status': 'act', 'valid': 'valid', 'type': 'HumanUser', 'name': 'Bertram Gilfoyle',1377 'id': 33},1378 {'status': 'act', 'valid': 'valid', 'type': 'HumanUser', 'name': 'Dinesh Chugtai',1379 'id': 57}]1380 :param dict entity: Entity to find followers of.1381 :returns: list of dicts representing each user following the entity1382 :rtype: list1383 :versionadded:1384 """1385 if not self.server_caps.version or self.server_caps.version < (5, 1, 22):1386 raise ShotgunError("Follow support requires server version 5.2 or "\1387 "higher, server is %s" % (self.server_caps.version,))1388 params = dict(1389 entity=entity1390 )1391 return self._call_rpc('followers', params)1392 def following(self, user, project=None, entity_type=None):1393 """1394 Return all entity instances a user is following.1395 Optionally, a project and/or entity_type can be supplied to restrict returned results.1396 >>> user = {"type": "HumanUser", "id": 1234}1397 >>> project = {"type": "Project", "id": 1234}1398 >>> entity_type = "Task"1399 >>> sg.following(user, project=project, entity_type=entity_type)1400 [{"type":"Task", "id":1},1401 {"type":"Task", "id":2},1402 {"type":"Task", "id":3}]1403 :param dict user: Find what this person is following.1404 :param dict project: Optional filter to only return results from a specific project.1405 :param str entity_type: Optional filter to only return results from one entity type.1406 :returns: list of dictionaries, each containing entity type & id's being followed.1407 :rtype: list1408 """1409 self.server_caps.ensure_user_following_support()1410 params = {1411 "user":user1412 }1413 if project:1414 params["project"] = project1415 if entity_type:1416 params["entity_type"] = entity_type1417 return self._call_rpc('following', params)1418 def schema_entity_read(self, project_entity=None):1419 """1420 Return all active entity types, their display names, and their visibility.1421 If the project parameter is specified, the schema visibility for the given project is1422 being returned. If the project parameter is omitted or set to ``None``, a full listing is1423 returned where per-project entity type visibility settings are not considered.1424 >>> sg.schema_entity_read()1425 {'ActionMenuItem': {'name': {'editable': False, 'value': 'Action Menu Item'},1426 'visible': {'editable': False, 'value': True}},1427 'ApiUser': {'name': {'editable': False, 'value': 'Script'},1428 'visible': {'editable': False, 'value': True}},1429 'AppWelcomeUserConnection': {'name': {'editable': False,1430 'value': 'App Welcome User Connection'},1431 'visible': {'editable': False, 'value': True}},1432 'Asset': {'name': {'editable': False, 'value': 'Asset'},1433 'visible': {'editable': False, 'value': True}},1434 'AssetAssetConnection': {'name': {'editable': False,1435 'value': 'Asset Asset Connection'},1436 'visible': {'editable': False, 'value': True}},1437 '...'1438 }1439 :param dict project_entity: Optional Project entity specifying which project to return1440 the listing for. If omitted or set to ``None``, per-project visibility settings are1441 not taken into consideration and the global list is returned. Example:1442 ``{'type': 'Project', 'id': 3}``1443 :returns: dict of Entity Type to dict containing the display name.1444 :rtype: dict1445 """1446 params = {}1447 params = self._add_project_param(params, project_entity)1448 if params:1449 return self._call_rpc("schema_entity_read", params)1450 else:1451 return self._call_rpc("schema_entity_read", None)1452 def schema_read(self, project_entity=None):1453 """1454 Get the schema for all fields on all entities.1455 .. note::1456 If ``project_entity`` is not specified, everything is reported as visible.1457 >>> sg.schema_read()1458 {'ActionMenuItem': {'created_at': {'data_type': {'editable': False, 'value': 'date_time'},1459 'description': {'editable': True, 'value': ''},1460 'editable': {'editable': False, 'value': False},1461 'entity_type': {'editable': False, 'value': 'ActionMenuItem'},1462 'mandatory': {'editable': False, 'value': False},1463 'name': {'editable': True, 'value': 'Date Created'},1464 'properties': {'default_value': {'editable': False, 'value': None},1465 'summary_default': {'editable': True, 'value': 'none'}},1466 'unique': {'editable': False, 'value': False},1467 'visible': {'editable': False, 'value': True}},1468 'created_by': {'data_type': {'editable': False,'value': 'entity'},1469 'description': {'editable': True,'value': ''},1470 'editable': {'editable': False,'value': False},1471 'entity_type': {'editable': False,'value': 'ActionMenuItem'},1472 'mandatory': {'editable': False,'value': False},1473 'name': {'editable': True,'value': 'Created by'},1474 'properties': {'default_value': {'editable': False,'value': None},1475 'summary_default': {'editable': True,'value': 'none'},1476 'valid_types': {'editable': True,'value': ['HumanUser','ApiUser']}},1477 'unique': {'editable': False,'value': False},1478 'visible': {'editable': False,'value': True}},1479 ...1480 ...1481 ...1482 ...1483 'Version': {'client_approved': {'data_type': {'editable': False,'value': 'checkbox'},1484 'description': {'editable': True,'value': ''},1485 'editable': {'editable': False,'value': True},1486 'entity_type': {'editable': False,'value': 'Version'},1487 'mandatory': {'editable': False,'value': False},1488 'name': {'editable': True,'value': 'Client Approved'},1489 'properties': {'default_value': {'editable': False,'value': False},1490 'summary_default': {'editable': False,'value': 'none'}},1491 'unique': {'editable': False,'value': False},1492 'visible': {'editable': False,'value': True}},1493 ...1494 ...1495 ...1496 ...1497 }1498 :param dict project_entity: Optional, Project entity specifying which project to return1499 the listing for. If omitted or set to ``None``, per-project visibility settings are1500 not taken into consideration and the global list is returned. Example:1501 ``{'type': 'Project', 'id': 3}``. Defaults to ``None``.1502 :returns: A nested dict object containing a key/value pair for all fields of all entity1503 types. Properties that are ``'editable': True``, can be updated using the1504 :meth:`~shotgun_api3.Shotgun.schema_field_update` method.1505 :rtype: dict1506 """1507 params = {}1508 params = self._add_project_param(params, project_entity)1509 if params:1510 return self._call_rpc("schema_read", params)1511 else:1512 return self._call_rpc("schema_read", None)1513 def schema_field_read(self, entity_type, field_name=None, project_entity=None):1514 """1515 Get schema for all fields on the specified entity type or just the field name specified1516 if provided.1517 .. note::1518 Unlike how the results of a :meth:`~shotgun_api3.Shotgun.find` can be pumped into a1519 :meth:`~shotgun_api3.Shotgun.create` or :meth:`~shotgun_api3.Shotgun.update`, the1520 results of :meth:`~shotgun_api3.Shotgun.schema_field_read` are not compatible with1521 the format used for :meth:`~shotgun_api3.Shotgun.schema_field_create` or1522 :meth:`~shotgun_api3.Shotgun.schema_field_update`. If you need to pipe the results1523 from :meth:`~shotgun_api3.Shotgun.schema_field_read` into a1524 :meth:`~shotgun_api3.Shotgun.schema_field_create` or1525 :meth:`~shotgun_api3.Shotgun.schema_field_update`, you will need to reformat the1526 data in your script.1527 .. note::1528 If you don't specify a ``project_entity``, everything is reported as visible.1529 >>> sg.schema_field_read('Asset', 'shots')1530 {'shots': {'data_type': {'editable': False, 'value': 'multi_entity'},1531 'description': {'editable': True, 'value': ''},1532 'editable': {'editable': False, 'value': True},1533 'entity_type': {'editable': False, 'value': 'Asset'},1534 'mandatory': {'editable': False, 'value': False},1535 'name': {'editable': True, 'value': 'Shots'},1536 'properties': {'default_value': {'editable': False,1537 'value': None},1538 'summary_default': {'editable': True,1539 'value': 'none'},1540 'valid_types': {'editable': True,1541 'value': ['Shot']}},1542 'unique': {'editable': False, 'value': False},1543 'visible': {'editable': False, 'value': True}}}1544 :param str entity_type: Entity type to get the schema for.1545 :param str field_name: Optional internal Shotgun name of the field to get the schema1546 definition for. If this parameter is excluded or set to ``None``, data structures of1547 all fields will be returned. Defaults to ``None``. Example: ``sg_temp_field``.1548 :param dict project_entity: Optional Project entity specifying which project to return1549 the listing for. If omitted or set to ``None``, per-project visibility settings are1550 not taken into consideration and the global list is returned. Example:1551 ``{'type': 'Project', 'id': 3}``1552 :returns: a nested dict object containing a key/value pair for the ``field_name`` specified1553 and its properties, or if no field_name is specified, for all the fields of the1554 ``entity_type``. Properties that are ``'editable': True``, can be updated using the1555 :meth:`~shotgun_api3.Shotgun.schema_field_update` method.1556 :rtype: dict1557 """1558 params = {1559 "type": entity_type,1560 }1561 if field_name:1562 params["field_name"] = field_name1563 params = self._add_project_param(params, project_entity)1564 return self._call_rpc("schema_field_read", params)1565 def schema_field_create(self, entity_type, data_type, display_name, properties=None):1566 """1567 Create a field for the specified entity type.1568 .. note::1569 If the internal Shotgun field name computed from the provided ``display_name`` already1570 exists, the internal Shotgun field name will automatically be appended with ``_1`` in1571 order to create a unique name. The integer suffix will be incremented by 1 until a1572 unique name is found.1573 >>> properties = {"summary_default": "count", "description": "Complexity breakdown of Asset"}1574 >>> sg.schema_field_create("Asset", "text", "Complexity", properties)1575 'sg_complexity'1576 :param str entity_type: Entity type to add the field to.1577 :param str data_type: Shotgun data type for the new field.1578 :param str display_name: Specifies the display name of the field you are creating. The1579 system name will be created from this display name and returned upon successful1580 creation.1581 :param dict properties: Dict of valid properties for the new field. Use this to specify1582 other field properties such as the 'description' or 'summary_default'.1583 :returns: The internal Shotgun name for the new field, this is different to the1584 ``display_name`` parameter passed in.1585 :rtype: str1586 """1587 params = {1588 "type" : entity_type,1589 "data_type" : data_type,1590 "properties" : [1591 {'property_name': 'name', 'value': display_name}1592 ]1593 }1594 params["properties"].extend(self._dict_to_list(properties,1595 key_name="property_name", value_name="value"))1596 return self._call_rpc("schema_field_create", params)1597 def schema_field_update(self, entity_type, field_name, properties):1598 """1599 Update the properties for the specified field on an entity.1600 .. note::1601 Although the property name may be the key in a nested dictionary, like1602 'summary_default', it is treated no differently than keys that are up1603 one level, like 'description'.1604 >>> properties = {"name": "Test Number Field Renamed", "summary_default": "sum",1605 ... "description": "this is only a test"}1606 >>> sg.schema_field_update("Asset", "sg_test_number", properties)1607 True1608 :param entity_type: Entity type of field to update.1609 :param field_name: Internal Shotgun name of the field to update.1610 :param properties: Dictionary with key/value pairs where the key is the property to be1611 updated and the value is the new value.1612 :returns: ``True`` if the field was updated.1613 :rtype: bool1614 """1615 params = {1616 "type" : entity_type,1617 "field_name" : field_name,1618 "properties": [1619 {"property_name" : k, "value" : v}1620 for k, v in (properties or {}).iteritems()1621 ]1622 }1623 return self._call_rpc("schema_field_update", params)1624 def schema_field_delete(self, entity_type, field_name):1625 """1626 Delete the specified field from the entity type.1627 >>> sg.schema_field_delete("Asset", "sg_temp_field")1628 True1629 :param str entity_type: Entity type to delete the field from.1630 :param str field_name: Internal Shotgun name of the field to delete.1631 :returns: ``True`` if the field was deleted.1632 :rtype: bool1633 """1634 params = {1635 "type" : entity_type,1636 "field_name" : field_name1637 }1638 return self._call_rpc("schema_field_delete", params)1639 def add_user_agent(self, agent):1640 """1641 Add agent to the user-agent header.1642 Appends agent to the user-agent string sent with every API request.1643 >>> sg.add_user_agent("my_tool 1.0")1644 :param str agent: string to append to user-agent.1645 """1646 self._user_agents.append(agent)1647 def reset_user_agent(self):1648 """1649 Reset user agent to the default value.1650 Example default user-agent::1651 shotgun-json (3.0.17); Python 2.6 (Mac); ssl OpenSSL 1.0.2d 9 Jul 2015 (validate)1652 """1653 ua_platform = "Unknown"1654 if self.client_caps.platform is not None:1655 ua_platform = self.client_caps.platform.capitalize()1656 # create ssl validation string based on settings1657 validation_str = "validate"1658 if self.config.no_ssl_validation:1659 validation_str = "no-validate"1660 self._user_agents = ["shotgun-json (%s)" % __version__,1661 "Python %s (%s)" % (self.client_caps.py_version, ua_platform),1662 "ssl %s (%s)" % (self.client_caps.ssl_version, validation_str)]1663 def set_session_uuid(self, session_uuid):1664 """1665 Set the browser session_uuid in the current Shotgun API instance.1666 When this is set, any events generated by the API will include the ``session_uuid`` value1667 on the corresponding EventLogEntries. If there is a current browser session open with1668 this ``session_uuid``, the browser will display updates for these events.1669 >>> sg.set_session_uuid("5a1d49b0-0c69-11e0-a24c-003048d17544")1670 :param str session_uuid: The uuid of the browser session to be updated.1671 """1672 self.config.session_uuid = session_uuid1673 return1674 def share_thumbnail(self, entities, thumbnail_path=None, source_entity=None,1675 filmstrip_thumbnail=False, **kwargs):1676 """1677 Associate a thumbnail with more than one Shotgun entity.1678 .. versionadded:: 3.0.91679 Requires Shotgun server v4.0.0+1680 Share the thumbnail from between entities without requiring uploading the thumbnail file1681 multiple times. You can use this in two ways:1682 1) Upload an image to set as the thumbnail on multiple entities.1683 2) Update multiple entities to point to an existing entity's thumbnail.1684 .. note::1685 When sharing a filmstrip thumbnail, it is required to have a static thumbnail in1686 place before the filmstrip will be displayed in the Shotgun web UI.1687 >>> thumb = '/data/show/ne2/100_110/anim/01.mlk-02b.jpg'1688 >>> e = [{'type': 'Version', 'id': 123}, {'type': 'Version', 'id': 456}]1689 >>> sg.share_thumbnail(entities=e, thumbnail_path=thumb)1690 42711691 >>> e = [{'type': 'Version', 'id': 123}, {'type': 'Version', 'id': 456}]1692 >>> sg.share_thumbnail(entities=e, source_entity={'type':'Version', 'id': 789})1693 42711694 :param list entities: The entities to update to point to the shared thumbnail provided in1695 standard entity dict format::1696 [{'type': 'Version', 'id': 123},1697 {'type': 'Version', 'id': 456}]1698 :param str thumbnail_path: The full path to the local thumbnail file to upload and share.1699 Required if ``source_entity`` is not provided.1700 :param dict source_entity: The entity whos thumbnail will be the source for sharing.1701 Required if ``source_entity`` is not provided.1702 :param bool filmstrip_thumbnail: ``True`` to share the filmstrip thumbnail. ``False`` to1703 share the static thumbnail. Defaults to ``False``.1704 :returns: ``id`` of the Attachment entity representing the source thumbnail that is shared.1705 :rtype: int1706 """1707 if not self.server_caps.version or self.server_caps.version < (4, 0, 0):1708 raise ShotgunError("Thumbnail sharing support requires server "\1709 "version 4.0 or higher, server is %s" % (self.server_caps.version,))1710 if not isinstance(entities, list) or len(entities) == 0:1711 raise ShotgunError("'entities' parameter must be a list of entity "\1712 "hashes and may not be empty")1713 for e in entities:1714 if not isinstance(e, dict) or 'id' not in e or 'type' not in e:1715 raise ShotgunError("'entities' parameter must be a list of "\1716 "entity hashes with at least 'type' and 'id' keys.\nInvalid "\1717 "entity: %s" % e)1718 if (not thumbnail_path and not source_entity) or \1719 (thumbnail_path and source_entity):1720 raise ShotgunError("You must supply either thumbnail_path OR "\1721 "source_entity.")1722 # upload thumbnail1723 if thumbnail_path:1724 source_entity = entities.pop(0)1725 if filmstrip_thumbnail:1726 thumb_id = self.upload_filmstrip_thumbnail(source_entity['type'],1727 source_entity['id'], thumbnail_path, **kwargs)1728 else:1729 thumb_id = self.upload_thumbnail(source_entity['type'],1730 source_entity['id'], thumbnail_path, **kwargs)1731 else:1732 if not isinstance(source_entity, dict) or 'id' not in source_entity \1733 or 'type' not in source_entity:1734 raise ShotgunError("'source_entity' parameter must be a dict "\1735 "with at least 'type' and 'id' keys.\nGot: %s (%s)" \1736 % (source_entity, type(source_entity)))1737 # only 1 entity in list and we already uploaded the thumbnail to it1738 if len(entities) == 0:1739 return thumb_id1740 # update entities with source_entity thumbnail1741 entities_str = []1742 for e in entities:1743 entities_str.append("%s_%s" % (e['type'], e['id']))1744 # format for post request1745 if filmstrip_thumbnail:1746 filmstrip_thumbnail = 11747 params = {1748 "entities" : ','.join(entities_str),1749 "source_entity": "%s_%s" % (source_entity['type'], source_entity['id']),1750 "filmstrip_thumbnail" : filmstrip_thumbnail,1751 }1752 url = urlparse.urlunparse((self.config.scheme, self.config.server,1753 "/upload/share_thumbnail", None, None, None))1754 result = self._send_form(url, params)1755 if not str(result).startswith("1"):1756 raise ShotgunError("Unable to share thumbnail: %s" % result)1757 else:1758 # clearing thumbnail returns no attachment_id1759 try:1760 attachment_id = int(str(result).split(":")[1].split("\n")[0])1761 except ValueError:1762 attachment_id = None1763 return attachment_id1764 def upload_thumbnail(self, entity_type, entity_id, path, **kwargs):1765 """1766 Upload a file from a local path and assign it as the thumbnail for the specified entity.1767 .. note::1768 Images will automatically be re-sized on the server to generate a size-appropriate image1769 file. However, the original file is retained as well and is accessible when you click1770 on the thumbnail image in the web UI. If you are using a local install of Shotgun and1771 have not enabled S3, this can eat up disk space if you're uploading really large source1772 images for your thumbnails.1773 You can un-set (aka clear) a thumbnail on an entity using the1774 :meth:`~shotgun_api3.Shotgun.update` method and setting the **image** field to ``None``.1775 This will also unset the ``filmstrip_thumbnail`` field if it is set.1776 Supported image file types include ``.jpg` and ``.png`` (preferred) but will also accept.1777 ``.gif```, ``.tif``, ``.tiff``, ``.bmp``, ``.exr``, ``.dpx``, and ``.tga``.1778 This method wraps over :meth:`~shotgun_api3.Shotgun.upload`. Additional keyword arguments1779 passed to this method will be forwarded to the :meth:`~shotgun_api3.Shotgun.upload` method.1780 :param str entity_type: Entity type to set the thumbnail for.1781 :param int entity_id: Id of the entity to set the thumbnail for.1782 :param str path: Full path to the thumbnail file on disk.1783 :returns: Id of the new attachment1784 """1785 return self.upload(entity_type, entity_id, path,1786 field_name="thumb_image", **kwargs)1787 def upload_filmstrip_thumbnail(self, entity_type, entity_id, path, **kwargs):1788 """1789 Upload filmstrip thumbnail to specified entity.1790 .. versionadded:: 3.0.91791 Requires Shotgun server v3.1.0+1792 Uploads a file from a local directory and assigns it as the filmstrip thumbnail for the1793 specified entity. The image must be a horizontal strip of any number of frames that are1794 exactly 240 pixels wide. Therefore the whole strip must be an exact multiple of 240 pixels1795 in width. The height can be anything (and will depend on the aspect ratio of the frames).1796 Any image file type that works for thumbnails will work for filmstrip thumbnails.1797 Filmstrip thumbnails will only be visible in the Thumbnail field on an entity if a1798 regular thumbnail image is also uploaded to the entity. The standard thumbnail is1799 displayed by default as the poster frame. Then, on hover, the filmstrip thumbnail is1800 displayed and updated based on your horizontal cursor position for scrubbing. On mouseout,1801 the default thumbnail is displayed again as the poster frame.1802 The url for a filmstrip thumbnail on an entity is available by querying for the1803 ``filmstrip_image field``.1804 You can un-set (aka clear) a thumbnail on an entity using the1805 :meth:`~shotgun_api3.Shotgun.update` method and setting the **image** field to ``None``.1806 This will also unset the ``filmstrip_thumbnail`` field if it is set.1807 This method wraps over :meth:`~shotgun_api3.Shotgun.upload`. Additional keyword arguments1808 passed to this method will be forwarded to the :meth:`~shotgun_api3.Shotgun.upload` method.1809 >>> filmstrip_thumbnail = '/data/show/ne2/100_110/anim/01.mlk-02b_filmstrip.jpg'1810 >>> sg.upload_filmstrip_thumbnail("Version", 27, filmstrip_thumbnail)1811 871812 :param str entity_type: Entity type to set the filmstrip thumbnail for.1813 :param int entity_id: Id of the entity to set the filmstrip thumbnail for.1814 :param str path: Full path to the filmstrip thumbnail file on disk.1815 :returns: Id of the new Attachment entity created for the filmstrip thumbnail1816 :rtype: int1817 """1818 if not self.server_caps.version or self.server_caps.version < (3, 1, 0):1819 raise ShotgunError("Filmstrip thumbnail support requires server version 3.1 or "\1820 "higher, server is %s" % (self.server_caps.version,))1821 return self.upload(entity_type, entity_id, path,1822 field_name="filmstrip_thumb_image", **kwargs)1823 def upload(self, entity_type, entity_id, path, field_name=None, display_name=None,1824 tag_list=None):1825 """1826 Upload a file to the specified entity.1827 Creates an Attachment entity for the file in Shotgun and links it to the specified entity.1828 You can optionally store the file in a field on the entity, change the display name, and1829 assign tags to the Attachment.1830 >>> mov_file = '/data/show/ne2/100_110/anim/01.mlk-02b.mov'1831 >>> sg.upload("Shot", 423, mov_file, field_name="sg_latest_quicktime",1832 ... display_name="Latest QT")1833 721834 :param str entity_type: Entity type to link the upload to.1835 :param int entity_id: Id of the entity to link the upload to.1836 :param str path: Full path to an existing non-empty file on disk to upload.1837 :param str field_name: The internal Shotgun field name on the entity to store the file in.1838 This field must be a File/Link field type.1839 :param str display_name: The display name to use for the file. Defaults to the file name.1840 :param str tag_list: comma-separated string of tags to assign to the file.1841 :returns: Id of the Attachment entity that was created for the image.1842 :rtype: int1843 """1844 # Basic validations of the file to upload.1845 path = os.path.abspath(os.path.expanduser(path or ""))1846 if not os.path.isfile(path):1847 raise ShotgunError("Path must be a valid file, got '%s'" % path)1848 if os.path.getsize(path) == 0:1849 raise ShotgunError("Path cannot be an empty file: '%s'" % path)1850 is_thumbnail = (field_name in ["thumb_image", "filmstrip_thumb_image", "image",1851 "filmstrip_image"])1852 # Version.sg_uploaded_movie is handled as a special case and uploaded1853 # directly to Cloud storage1854 if self.server_info.get("s3_direct_uploads_enabled", False) \1855 and entity_type == "Version" and field_name == "sg_uploaded_movie":1856 return self._upload_to_storage(entity_type, entity_id, path, field_name, display_name,1857 tag_list, is_thumbnail)1858 else:1859 return self._upload_to_sg(entity_type, entity_id, path, field_name, display_name,1860 tag_list, is_thumbnail)1861 def _upload_to_storage(self, entity_type, entity_id, path, field_name, display_name,1862 tag_list, is_thumbnail):1863 """1864 Internal function to upload a file to the Cloud storage and link it to the specified entity.1865 :param str entity_type: Entity type to link the upload to.1866 :param int entity_id: Id of the entity to link the upload to.1867 :param str path: Full path to an existing non-empty file on disk to upload.1868 :param str field_name: The internal Shotgun field name on the entity to store the file in.1869 This field must be a File/Link field type.1870 :param str display_name: The display name to use for the file. Defaults to the file name.1871 :param str tag_list: comma-separated string of tags to assign to the file.1872 :param bool is_thumbnail: indicates if the attachment is a thumbnail.1873 :returns: Id of the Attachment entity that was created for the image.1874 :rtype: int1875 """1876 filename = os.path.basename(path)1877 # Step 1: get the upload url1878 is_multipart_upload = (os.path.getsize(path) > self._MULTIPART_UPLOAD_CHUNK_SIZE)1879 upload_info = self._get_attachment_upload_info(is_thumbnail, filename, is_multipart_upload)1880 # Step 2: upload the file1881 # We upload large files in multiple parts because it is more robust1882 # (and required when using S3 storage)1883 if is_multipart_upload:1884 self._multipart_upload_file_to_storage(path, upload_info)1885 else:1886 self._upload_file_to_storage(path, upload_info["upload_url"])1887 # Step 3: create the attachment1888 url = urlparse.urlunparse((self.config.scheme, self.config.server,1889 "/upload/api_link_file", None, None, None))1890 params = {1891 "entity_type" : entity_type,1892 "entity_id" : entity_id,1893 "upload_link_info": upload_info['upload_info']1894 }1895 params.update(self._auth_params())1896 if is_thumbnail:1897 if field_name == "filmstrip_thumb_image" or field_name == "filmstrip_image":1898 params["filmstrip"] = True1899 else:1900 if display_name is None:1901 display_name = filename1902 # we allow linking to nothing for generic reference use cases1903 if field_name is not None:1904 params["field_name"] = field_name1905 params["display_name"] = display_name1906 # None gets converted to a string and added as a tag...1907 if tag_list:1908 params["tag_list"] = tag_list1909 result = self._send_form(url, params)1910 if not str(result).startswith("1"):1911 raise ShotgunError("Could not upload file successfully, but " \1912 "not sure why.\nPath: %s\nUrl: %s\nError: %s" % (1913 path, url, str(result)))1914 LOG.debug("Attachment linked to content on Cloud storage")1915 attachment_id = int(str(result).split(":")[1].split("\n")[0])1916 return attachment_id1917 def _upload_to_sg(self, entity_type, entity_id, path, field_name, display_name,1918 tag_list, is_thumbnail):1919 """1920 Internal function to upload a file to Shotgun and link it to the specified entity.1921 :param str entity_type: Entity type to link the upload to.1922 :param int entity_id: Id of the entity to link the upload to.1923 :param str path: Full path to an existing non-empty file on disk to upload.1924 :param str field_name: The internal Shotgun field name on the entity to store the file in.1925 This field must be a File/Link field type.1926 :param str display_name: The display name to use for the file. Defaults to the file name.1927 :param str tag_list: comma-separated string of tags to assign to the file.1928 :param bool is_thumbnail: indicates if the attachment is a thumbnail.1929 :returns: Id of the Attachment entity that was created for the image.1930 :rtype: int1931 """1932 params = {1933 "entity_type" : entity_type,1934 "entity_id" : entity_id,1935 }1936 params.update(self._auth_params())1937 if is_thumbnail:1938 url = urlparse.urlunparse((self.config.scheme, self.config.server,1939 "/upload/publish_thumbnail", None, None, None))1940 params["thumb_image"] = open(path, "rb")1941 if field_name == "filmstrip_thumb_image" or field_name == "filmstrip_image":1942 params["filmstrip"] = True1943 else:1944 url = urlparse.urlunparse((self.config.scheme, self.config.server,1945 "/upload/upload_file", None, None, None))1946 if display_name is None:1947 display_name = os.path.basename(path)1948 # we allow linking to nothing for generic reference use cases1949 if field_name is not None:1950 params["field_name"] = field_name1951 params["display_name"] = display_name1952 # None gets converted to a string and added as a tag...1953 if tag_list:1954 params["tag_list"] = tag_list1955 params["file"] = open(path, "rb")1956 result = self._send_form(url, params)1957 if not str(result).startswith("1"):1958 raise ShotgunError("Could not upload file successfully, but "\1959 "not sure why.\nPath: %s\nUrl: %s\nError: %s" % (1960 path, url, str(result)))1961 attachment_id = int(str(result).split(":")[1].split("\n")[0])1962 return attachment_id1963 def _get_attachment_upload_info(self, is_thumbnail, filename, is_multipart_upload):1964 """1965 Internal function to get the information needed to upload a file to Cloud storage.1966 :param bool is_thumbnail: indicates if the attachment is a thumbnail.1967 :param str filename: name of the file that will be uploaded.1968 :param bool is_multipart_upload: Indicates if we want multi-part upload information back.1969 :returns: dictionary containing upload details from the server.1970 These details are used throughout the upload process.1971 :rtype: dict1972 """1973 if is_thumbnail:1974 upload_type = "Thumbnail"1975 else:1976 upload_type = "Attachment"1977 params = {1978 "upload_type" : upload_type,1979 "filename" : filename1980 }1981 params["multipart_upload"] = is_multipart_upload1982 upload_url = "/upload/api_get_upload_link_info"1983 url = urlparse.urlunparse((self.config.scheme, self.config.server,1984 upload_url, None, None, None))1985 upload_info = self._send_form(url, params)1986 if not str(upload_info).startswith("1"):1987 raise ShotgunError("Could not get upload_url but " \1988 "not sure why.\nPath: %s\nUrl: %s\nError: %s" % (1989 filename, url, str(upload_info)))1990 LOG.debug("Completed rpc call to %s" % (upload_url))1991 upload_info_parts = str(upload_info).split("\n")1992 return {1993 "upload_url" : upload_info_parts[1],1994 "timestamp" : upload_info_parts[2],1995 "upload_type" : upload_info_parts[3],1996 "upload_id": upload_info_parts[4],1997 "upload_info" : upload_info1998 }1999 def download_attachment(self, attachment=False, file_path=None, attachment_id=None):2000 """2001 Download the file associated with a Shotgun Attachment.2002 >>> version = sg.find_one("Version", [["id", "is", 7115]], ["sg_uploaded_movie"])2003 >>> local_file_path = "/var/tmp/%s" % version["sg_uploaded_movie"]["name"]2004 >>> sg.download_attachment(version["sg_uploaded_movie"], file_path=local_file_path)2005 /var/tmp/100b_scene_output_v032.mov2006 .. warning::2007 On older (< v5.1.0) Shotgun versions, non-downloadable files2008 on Shotgun don't raise exceptions, they cause a server error which2009 returns a 200 with the page content.2010 :param dict attachment: Usually a dictionary representing an Attachment entity.2011 The dictionary should have a ``url`` key that specifies the download url.2012 Optionally, the dictionary can be a standard entity hash format with ``id`` and2013 ``type`` keys as long as ``"type"=="Attachment"``. This is only supported for2014 backwards compatibility (#22150).2015 If an int value is passed in, the Attachment entity with the matching id will2016 be downloaded from the Shotgun server.2017 :param str file_path: Optional file path to write the data directly to local disk. This2018 avoids loading all of the data in memory and saves the file locally at the given path.2019 :param id attachment_id: (deprecated) Optional ``id`` of the Attachment entity in Shotgun to2020 download.2021 .. note:2022 This parameter exists only for backwards compatibility for scripts specifying2023 the parameter with keywords.2024 :returns: If ``file_path`` is provided, returns the path to the file on disk. If2025 ``file_path`` is ``None``, returns the actual data of the file as a string.2026 :rtype: str2027 """2028 # backwards compatibility when passed via keyword argument2029 if attachment is False:2030 if type(attachment_id) == int:2031 attachment = attachment_id2032 else:2033 raise TypeError("Missing parameter 'attachment'. Expected a "\2034 "dict, int, NoneType value or"\2035 "an int for parameter attachment_id")2036 # write to disk2037 if file_path:2038 try:2039 fp = open(file_path, 'wb')2040 except IOError, e:2041 raise IOError("Unable to write Attachment to disk using "\2042 "file_path. %s" % e)2043 url = self.get_attachment_download_url(attachment)2044 if url is None:2045 return None2046 # We only need to set the auth cookie for downloads from Shotgun server2047 if self.config.server in url:2048 self.set_up_auth_cookie()2049 try:2050 request = urllib2.Request(url)2051 request.add_header('user-agent', "; ".join(self._user_agents))2052 req = urllib2.urlopen(request)2053 if file_path:2054 shutil.copyfileobj(req, fp)2055 else:2056 attachment = req.read()2057 # 400 [sg] Attachment id doesn't exist or is a local file2058 # 403 [s3] link is invalid2059 except urllib2.URLError, e:2060 if file_path:2061 fp.close()2062 err = "Failed to open %s\n%s" % (url, e)2063 if hasattr(e, 'code'):2064 if e.code == 400:2065 err += "\nAttachment may not exist or is a local file?"2066 elif e.code == 403:2067 # Only parse the body if it is an Amazon S3 url.2068 if url.find('s3.amazonaws.com') != -1 \2069 and e.headers['content-type'] == 'application/xml':2070 body = e.readlines()2071 if body:2072 xml = ''.join(body)2073 # Once python 2.4 support is not needed we can think about using2074 # elementtree. The doc is pretty small so this shouldn't be an issue.2075 match = re.search('<Message>(.*)</Message>', xml)2076 if match:2077 err += ' - %s' % (match.group(1))2078 raise ShotgunFileDownloadError(err)2079 else:2080 if file_path:2081 if not fp.closed:2082 fp.close()2083 return file_path2084 else:2085 return attachment2086 def set_up_auth_cookie(self):2087 """2088 Set up urllib2 with a cookie for authentication on the Shotgun instance.2089 Looks up session token and sets that in a cookie in the :mod:`urllib2` handler. This is2090 used internally for downloading attachments from the Shotgun server.2091 """2092 sid = self.get_session_token()2093 cj = cookielib.LWPCookieJar()2094 c = cookielib.Cookie('0', '_session_id', sid, None, False,2095 self.config.server, False, False, "/", True, False, None, True,2096 None, None, {})2097 cj.set_cookie(c)2098 cookie_handler = urllib2.HTTPCookieProcessor(cj)2099 opener = self._build_opener(cookie_handler)2100 urllib2.install_opener(opener)2101 def get_attachment_download_url(self, attachment):2102 """2103 Return the URL for downloading provided Attachment.2104 :param mixed attachment: Usually a dict representing An Attachment entity in Shotgun to2105 return the download url for. If the ``url`` key is present, it will be used as-is for2106 the download url. If the ``url`` key is not present, a url will be constructed pointing2107 at the current Shotgun server for downloading the Attachment entity using the ``id``.2108 If ``None`` is passed in, it is silently ignored in order to avoid raising an error when2109 results from a :meth:`~shotgun_api3.Shotgun.find` are passed off to2110 :meth:`~shotgun_api3.Shotgun.download_attachment`2111 .. note::2112 Support for passing in an int representing the Attachment ``id`` is deprecated2113 .. todo::2114 Support for a standard entity hash should be removed: #221502115 :returns: the download URL for the Attachment or ``None`` if ``None`` was passed to2116 ``attachment`` parameter.2117 :rtype: str2118 """2119 attachment_id = None2120 if isinstance(attachment, int):2121 attachment_id = attachment2122 elif isinstance(attachment, dict):2123 try:2124 url = attachment['url']2125 except KeyError:2126 if ('id' in attachment and 'type' in attachment and2127 attachment['type'] == 'Attachment'):2128 attachment_id = attachment['id']2129 else:2130 raise ValueError("Missing 'url' key in Attachment dict")2131 elif attachment is None:2132 url = None2133 else:2134 raise TypeError("Unable to determine download url. Expected "\2135 "dict, int, or NoneType. Instead got %s" % type(attachment))2136 if attachment_id:2137 url = urlparse.urlunparse((self.config.scheme, self.config.server,2138 "/file_serve/attachment/%s" % urllib.quote(str(attachment_id)),2139 None, None, None))2140 return url2141 def authenticate_human_user(self, user_login, user_password, auth_token=None):2142 """2143 Authenticate Shotgun HumanUser.2144 Authenticates a user given the login, password, and optionally, one-time auth token (when2145 two-factor authentication is required). The user must be a ``HumanUser`` entity and the2146 account must be active.2147 >>> sg.authenticate_human_user("rhendriks", "c0mPre$Hi0n", None)2148 {"type": "HumanUser", "id": 123, "name": "Richard Hendriks"}2149 :param str user_login: Login name of Shotgun HumanUser2150 :param str user_password: Password for Shotgun HumanUser2151 :param str auth_token: One-time token required to authenticate Shotgun HumanUser2152 when two-factor authentication is turned on.2153 :returns: Standard Shotgun dictionary representing the HumanUser if authentication2154 succeeded. ``None`` if authentication failed for any reason.2155 :rtype: dict2156 """2157 if not user_login:2158 raise ValueError('Please supply a username to authenticate.')2159 if not user_password:2160 raise ValueError('Please supply a password for the user.')2161 # Override permissions on Config obj2162 original_login = self.config.user_login2163 original_password = self.config.user_password2164 original_auth_token = self.config.auth_token2165 self.config.user_login = user_login2166 self.config.user_password = user_password2167 self.config.auth_token = auth_token2168 try:2169 data = self.find_one('HumanUser', [['sg_status_list', 'is', 'act'],2170 ['login', 'is', user_login]],2171 ['id', 'login'], '', 'all')2172 # Set back to default - There finally and except cannot be used together in python2.42173 self.config.user_login = original_login2174 self.config.user_password = original_password2175 self.config.auth_token = original_auth_token2176 return data2177 except Fault:2178 # Set back to default - There finally and except cannot be used together in python2.42179 self.config.user_login = original_login2180 self.config.user_password = original_password2181 self.config.auth_token = original_auth_token2182 except:2183 # Set back to default - There finally and except cannot be used together in python2.42184 self.config.user_login = original_login2185 self.config.user_password = original_password2186 self.config.auth_token = original_auth_token2187 raise2188 def update_project_last_accessed(self, project, user=None):2189 """2190 Update a Project's ``last_accessed_by_current_user`` field to the current timestamp.2191 This helps keep track of the recent Projects each user has worked on and enables scripts2192 and apps to use this information to display "Recent Projects" for users as a convenience.2193 .. versionadded::2194 Requires Shotgun v5.3.20+2195 >>> sg.update_project_last_accessed({"type": "Project", "id": 66},2196 ... {"type": "HumanUser", "id": 43})2197 :param dict project: Standard Project entity dictionary2198 :param dict user: Standard user entity dictionary. This is optional if the current API2199 instance is using user-based authenitcation, or has specified ``sudo_as_login``. In2200 these cases, if ``user`` is not provided, the ``sudo_as_login`` value or ``login``2201 value from the current instance will be used instead.2202 """2203 if self.server_caps.version and self.server_caps.version < (5, 3, 20):2204 raise ShotgunError("update_project_last_accessed requires server version 5.3.20 or "2205 "higher, server is %s" % (self.server_caps.version,))2206 if not user:2207 # Try to use sudo as user if present2208 if self.config.sudo_as_login:2209 user = self.find_one('HumanUser', [['login', 'is', self.config.sudo_as_login]])2210 # Try to use login if present2211 if self.config.user_login:2212 user = self.find_one('HumanUser', [['login', 'is', self.config.user_login]])2213 params = { "project_id": project['id'], }2214 if user:2215 params['user_id'] = user['id']2216 record = self._call_rpc("update_project_last_accessed_by_current_user", params)2217 self._parse_records(record)[0]2218 def note_thread_read(self, note_id, entity_fields=None):2219 """2220 Return the full conversation for a given note, including Replies and Attachments.2221 Returns a complex data structure on the following form::2222 [{'content': 'Please add more awesomeness to the color grading.',2223 'created_at': '2015-07-14 21:33:28 UTC',2224 'created_by': {'id': 38,2225 'name': 'John Pink',2226 'status': 'act',2227 'type': 'HumanUser',2228 'valid': 'valid'},2229 'id': 6013,2230 'type': 'Note'},2231 {'created_at': '2015-07-14 21:33:32 UTC',2232 'created_by': {'id': 38,2233 'name': 'John Pink',2234 'status': 'act',2235 'type': 'HumanUser',2236 'valid': 'valid'},2237 'id': 159,2238 'type': 'Attachment'},2239 {'content': 'More awesomeness added',2240 'created_at': '2015-07-14 21:54:51 UTC',2241 'id': 5,2242 'type': 'Reply',2243 'user': {'id': 38,2244 'name': 'David Blue',2245 'status': 'act',2246 'type': 'HumanUser',2247 'valid': 'valid'}}]2248 The list is returned in descending chronological order.2249 If you wish to include additional fields beyond the ones that are2250 returned by default, you can specify these in an entity_fields2251 dictionary. This dictionary should be keyed by entity type and each2252 key should contain a list of fields to retrieve, for example::2253 { "Note": ["created_by.HumanUser.image",2254 "addressings_to",2255 "playlist",2256 "user" ],2257 "Reply": ["content"],2258 "Attachment": ["filmstrip_image",2259 "local_storage",2260 "this_file",2261 "image"]2262 }2263 :param int note_id: The id for the note to be retrieved2264 :param dict entity_fields: Additional fields to retrieve as part of the request.2265 See above for details.2266 :returns: A list of dictionaries. See above for example.2267 :rtype: list2268 """2269 if self.server_caps.version and self.server_caps.version < (6, 2, 0):2270 raise ShotgunError("note_thread requires server version 6.2.0 or "\2271 "higher, server is %s" % (self.server_caps.version,))2272 entity_fields = entity_fields or {}2273 if not isinstance(entity_fields, dict):2274 raise ValueError("entity_fields parameter must be a dictionary")2275 params = { "note_id": note_id, "entity_fields": entity_fields }2276 record = self._call_rpc("note_thread_contents", params)2277 result = self._parse_records(record)2278 return result2279 def text_search(self, text, entity_types, project_ids=None, limit=None):2280 """2281 Search across the specified entity types for the given text.2282 This method can be used to implement auto completion or a Shotgun global search. The method2283 requires a text input phrase that is at least three characters long, or an exception will2284 be raised.2285 Several ways to limit the results of the query are available:2286 - Using the ``project_ids`` parameter, you can provide a list of Project ids to search2287 across. Leaving this at its default value of ``None`` will search across all Shotgun data.2288 - You need to define which subset of entity types to search using the ``entity_types``2289 parameter. Each of these entity types can be associated with a filter query to further2290 reduce the list of matches. The filter list is using the standard filter syntax used by2291 for example the :meth:`~shotgun_api3.Shotgun.find` method.2292 **Example: Constrain the search to all Tasks but Character Assets only**2293 >>> entity_types = {2294 ... "Asset": [["sg_asset_type", "is", "Character"]],2295 ... "Task": []2296 ... }2297 >>> sg.text_search("bunny", entity_types)2298 {'matches': [{'id': 734,2299 'type': 'Asset',2300 'name': 'Bunny',2301 'project_id': 65,2302 'image': 'https://...',2303 'links': ['', ''],2304 'status': 'fin'},2305 ...2306 {'id': 558,2307 'type': 'Task'2308 'name': 'FX',2309 'project_id': 65,2310 'image': 'https://...',2311 'links': ['Shot', 'bunny_010_0010'],2312 'status': 'fin'}],2313 'terms': ['bunny']}2314 The links field will contain information about any linked entity. This is useful when, for2315 example, presenting Tasks and you want to display what Shot or Asset the Task is associated2316 with.2317 :param str text: Text to search for. This must be at least three characters long, or an2318 exception will be raised.2319 :param dict entity_types: Dictionary to specify which entity types to search across. See2320 above for usage examples.2321 :param list project_ids: List of Projects to search. By default, all projects will be2322 searched.2323 :param int limit: Specify the maximum number of matches to return.2324 :returns: A complex dictionary structure, see above for example.2325 :rtype: dict2326 """2327 if self.server_caps.version and self.server_caps.version < (6, 2, 0):2328 raise ShotgunError("auto_complete requires server version 6.2.0 or "\2329 "higher, server is %s" % (self.server_caps.version,))2330 # convert entity_types structure into the form2331 # that the API endpoint expects2332 if not isinstance(entity_types, dict):2333 raise ValueError("entity_types parameter must be a dictionary")2334 api_entity_types = {}2335 for (entity_type, filter_list) in entity_types.iteritems():2336 if isinstance(filter_list, (list, tuple)):2337 resolved_filters = _translate_filters(filter_list, filter_operator=None)2338 api_entity_types[entity_type] = resolved_filters2339 else:2340 raise ValueError("value of entity_types['%s'] must "2341 "be a list or tuple." % entity_type)2342 project_ids = project_ids or []2343 params = { "text": text,2344 "entity_types": api_entity_types,2345 "project_ids": project_ids,2346 "max_results": limit }2347 record = self._call_rpc("query_display_name_cache", params)2348 result = self._parse_records(record)[0]2349 return result2350 def activity_stream_read(self, entity_type, entity_id, entity_fields=None, min_id=None,2351 max_id=None, limit=None):2352 """2353 Retrieve activity stream data from Shotgun.2354 This data corresponds to the data that is displayed in the2355 Activity tab for an entity in the Shotgun Web UI.2356 A complex data structure on the following form will be2357 returned from Shotgun::2358 {'earliest_update_id': 50,2359 'entity_id': 65,2360 'entity_type': 'Project',2361 'latest_update_id': 79,2362 'updates': [{'created_at': '2015-07-15 11:06:55 UTC',2363 'created_by': {'id': 38,2364 'image': '6641',2365 'name': 'John Smith',2366 'status': 'act',2367 'type': 'HumanUser'},2368 'id': 79,2369 'meta': {'entity_id': 6004,2370 'entity_type': 'Version',2371 'type': 'new_entity'},2372 'primary_entity': {'id': 6004,2373 'name': 'Review_turntable_v2',2374 'status': 'rev',2375 'type': 'Version'},2376 'read': False,2377 'update_type': 'create'},2378 {...},2379 ]2380 }2381 The main payload of the return data can be found inside the 'updates'2382 key, containing a list of dictionaries. This list is always returned2383 in descending date order. Each item may contain different fields2384 depending on their update type. The primary_entity key represents the2385 main Shotgun entity that is associated with the update. By default,2386 this entity is returned with a set of standard fields. By using the2387 entity_fields parameter, you can extend the returned data to include2388 additional fields. If for example you wanted to return the asset type2389 for all assets and the linked sequence for all Shots, pass the2390 following entity_fields::2391 {"Shot": ["sg_sequence"], "Asset": ["sg_asset_type"]}2392 Deep queries can be used in this syntax if you want to2393 traverse into connected data.2394 :param str entity_type: Entity type to retrieve activity stream for2395 :param int entity_id: Entity id to retrieve activity stream for2396 :param list entity_fields: List of additional fields to include.2397 See above for details2398 :param int max_id: Do not retrieve ids greater than this id.2399 This is useful when implementing paging.2400 :param int min_id: Do not retrieve ids lesser than this id.2401 This is useful when implementing caching of2402 the event stream data and you want to2403 "top up" an existing cache.2404 :param int limit: Limit the number of returned records. If not specified,2405 the system default will be used.2406 :returns: A complex activity stream data structure. See above for details.2407 :rtype: dict2408 """2409 if self.server_caps.version and self.server_caps.version < (6, 2, 0):2410 raise ShotgunError("activity_stream requires server version 6.2.0 or "\2411 "higher, server is %s" % (self.server_caps.version,))2412 # set up parameters to send to server.2413 entity_fields = entity_fields or {}2414 if not isinstance(entity_fields, dict):2415 raise ValueError("entity_fields parameter must be a dictionary")2416 params = { "type": entity_type,2417 "id": entity_id,2418 "max_id": max_id,2419 "min_id": min_id,2420 "limit": limit,2421 "entity_fields": entity_fields }2422 record = self._call_rpc("activity_stream", params)2423 result = self._parse_records(record)[0]2424 return result2425 def nav_expand(self, path, seed_entity_field=None, entity_fields=None):2426 """2427 Expand the navigation hierarchy for the supplied path.2428 .. warning::2429 This is an experimental method that is not officially part of the2430 python-api. Usage of this method is discouraged. This method's name,2431 arguments, and argument types may change at any point.2432 """2433 return self._call_rpc(2434 "nav_expand",2435 {2436 "path":path,2437 "seed_entity_field": seed_entity_field,2438 "entity_fields": entity_fields2439 }2440 )2441 def nav_search_string(self, root_path, search_string, seed_entity_field=None):2442 """2443 Search function adapted to work with the navigation hierarchy.2444 .. warning::2445 This is an experimental method that is not officially part of the2446 python-api. Usage of this method is discouraged. This method's name,2447 arguments, and argument types may change at any point.2448 """2449 return self._call_rpc(2450 "nav_search",2451 {2452 "root_path":root_path,2453 "seed_entity_field": seed_entity_field,2454 "search_criteria": { "search_string": search_string }2455 }2456 )2457 def nav_search_entity(self, root_path, entity, seed_entity_field=None):2458 """2459 Search function adapted to work with the navigation hierarchy.2460 .. warning::2461 This is an experimental method that is not officially part of the2462 python-api. Usage of this method is discouraged. This method's name,2463 arguments, and argument types may change at any point.2464 """2465 return self._call_rpc(2466 "nav_search",2467 {2468 "root_path": root_path,2469 "seed_entity_field": seed_entity_field,2470 "search_criteria": {"entity": entity }2471 }2472 )2473 def get_session_token(self):2474 """2475 Get the session token associated with the current session.2476 If a session token has already been established, this is returned, otherwise a new one is2477 generated on the server and returned.2478 >>> sg.get_session_token()2479 dd638be7d07c39fa73d935a775558a502480 :returns: String containing a session token.2481 :rtype: str2482 """2483 if self.config.session_token:2484 return self.config.session_token2485 rv = self._call_rpc("get_session_token", None)2486 session_token = (rv or {}).get("session_id")2487 if not session_token:2488 raise RuntimeError("Could not extract session_id from %s", rv)2489 self.config.session_token = session_token2490 return session_token2491 def _build_opener(self, handler):2492 """2493 Build urllib2 opener with appropriate proxy handler.2494 """2495 if self.config.proxy_handler:2496 opener = urllib2.build_opener(self.config.proxy_handler, handler)2497 else:2498 opener = urllib2.build_opener(handler)2499 return opener2500 def _turn_off_ssl_validation(self):2501 """2502 Turn off SSL certificate validation.2503 """2504 global NO_SSL_VALIDATION2505 self.config.no_ssl_validation = True2506 NO_SSL_VALIDATION = True2507 # reset ssl-validation in user-agents2508 self._user_agents = ["ssl %s (no-validate)" % self.client_caps.ssl_version2509 if ua.startswith("ssl ") else ua2510 for ua in self._user_agents]2511 # Deprecated methods from old wrapper2512 def schema(self, entity_type):2513 """2514 .. deprecated:: 3.0.02515 Use :meth:`~shotgun_api3.Shotgun.schema_field_read` instead.2516 """2517 raise ShotgunError("Deprecated: use schema_field_read('type':'%s') "2518 "instead" % entity_type)2519 def entity_types(self):2520 """2521 .. deprecated:: 3.0.02522 Use :meth:`~shotgun_api3.Shotgun.schema_entity_read` instead.2523 """2524 raise ShotgunError("Deprecated: use schema_entity_read() instead")2525 # ========================================================================2526 # RPC Functions2527 def _call_rpc(self, method, params, include_auth_params=True, first=False):2528 """2529 Call the specified method on the Shotgun Server sending the supplied payload.2530 """2531 LOG.debug("Starting rpc call to %s with params %s" % (2532 method, params))2533 params = self._transform_outbound(params)2534 payload = self._build_payload(method, params,2535 include_auth_params=include_auth_params)2536 encoded_payload = self._encode_payload(payload)2537 req_headers = {2538 "content-type" : "application/json; charset=utf-8",2539 "connection" : "keep-alive"2540 }2541 http_status, resp_headers, body = self._make_call("POST",2542 self.config.api_path, encoded_payload, req_headers)2543 LOG.debug("Completed rpc call to %s" % (method))2544 try:2545 self._parse_http_status(http_status)2546 except ProtocolError, e:2547 e.headers = resp_headers2548 # 403 is returned with custom error page when api access is blocked2549 if e.errcode == 403:2550 e.errmsg += ": %s" % body2551 raise2552 response = self._decode_response(resp_headers, body)2553 self._response_errors(response)2554 response = self._transform_inbound(response)2555 if not isinstance(response, dict) or "results" not in response:2556 return response2557 results = response.get("results")2558 if first and isinstance(results, list):2559 return results[0]2560 return results2561 def _auth_params(self):2562 """2563 Return a dictionary of the authentication parameters being used.2564 """2565 # Used to authenticate HumanUser credentials2566 if self.config.user_login and self.config.user_password:2567 auth_params = {2568 "user_login" : str(self.config.user_login),2569 "user_password" : str(self.config.user_password),2570 }2571 if self.config.auth_token:2572 auth_params["auth_token"] = str(self.config.auth_token)2573 # Use script name instead2574 elif self.config.script_name and self.config.api_key:2575 auth_params = {2576 "script_name" : str(self.config.script_name),2577 "script_key" : str(self.config.api_key),2578 }2579 # Authenticate using session_id2580 elif self.config.session_token:2581 if self.server_caps.version and self.server_caps.version < (5, 3, 0):2582 raise ShotgunError("Session token based authentication requires server version "2583 "5.3.0 or higher, server is %s" % (self.server_caps.version,))2584 auth_params = {"session_token" : str(self.config.session_token)}2585 # Request server side to raise exception for expired sessions.2586 # This was added in as part of Shotgun 5.4.42587 if self.server_caps.version and self.server_caps.version > (5, 4, 3):2588 auth_params["reject_if_expired"] = True2589 else:2590 raise ValueError("invalid auth params")2591 if self.config.session_uuid:2592 auth_params["session_uuid"] = self.config.session_uuid2593 # Make sure sudo_as_login is supported by server version2594 if self.config.sudo_as_login:2595 if self.server_caps.version and self.server_caps.version < (5, 3, 12):2596 raise ShotgunError("Option 'sudo_as_login' requires server version 5.3.12 or "\2597 "higher, server is %s" % (self.server_caps.version,))2598 auth_params["sudo_as_login"] = self.config.sudo_as_login2599 if self.config.extra_auth_params:2600 auth_params.update(self.config.extra_auth_params)2601 return auth_params2602 def _sanitize_auth_params(self, params):2603 """2604 Given an authentication parameter dictionary, sanitize any sensitive2605 information and return the sanitized dict copy.2606 """2607 sanitized_params = copy.copy(params)2608 for k in ['user_password', 'script_key', 'session_token']:2609 if k in sanitized_params:2610 sanitized_params[k] = '********'2611 return sanitized_params2612 def _build_payload(self, method, params, include_auth_params=True):2613 """2614 Build the payload to be send to the rpc endpoint.2615 """2616 if not method:2617 raise ValueError("method is empty")2618 call_params = []2619 if include_auth_params:2620 auth_params = self._auth_params()2621 call_params.append(auth_params)2622 if params:2623 call_params.append(params)2624 return {2625 "method_name" : method,2626 "params" : call_params2627 }2628 def _encode_payload(self, payload):2629 """2630 Encode the payload to a string to be passed to the rpc endpoint.2631 The payload is json encoded as a unicode string if the content2632 requires it. The unicode string is then encoded as 'utf-8' as it must2633 be in a single byte encoding to go over the wire.2634 """2635 wire = json.dumps(payload, ensure_ascii=False)2636 if isinstance(wire, unicode):2637 return wire.encode("utf-8")2638 return wire2639 def _make_call(self, verb, path, body, headers):2640 """2641 Make an HTTP call to the server.2642 Handles retry and failure.2643 """2644 attempt = 02645 req_headers = {}2646 req_headers["user-agent"] = "; ".join(self._user_agents)2647 if self.config.authorization:2648 req_headers["Authorization"] = self.config.authorization2649 req_headers.update(headers or {})2650 body = body or None2651 max_rpc_attempts = self.config.max_rpc_attempts2652 while (attempt < max_rpc_attempts):2653 attempt += 12654 try:2655 return self._http_request(verb, path, body, req_headers)2656 except SSLHandshakeError, e:2657 # Test whether the exception is due to the fact that this is an older version of2658 # Python that cannot validate certificates encrypted with SHA-2. If it is, then2659 # fall back on disabling the certificate validation and try again - unless the2660 # SHOTGUN_FORCE_CERTIFICATE_VALIDATION environment variable has been set by the2661 # user. In that case we simply raise the exception. Any other exceptions simply2662 # get raised as well.2663 #2664 # For more info see:2665 # http://blog.shotgunsoftware.com/2016/01/important-ssl-certificate-renewal-and.html2666 #2667 # SHA-2 errors look like this:2668 # [Errno 1] _ssl.c:480: error:0D0C50A1:asn1 encoding routines:ASN1_item_verify:2669 # unknown message digest algorithm2670 #2671 # Any other exceptions simply get raised.2672 if not str(e).endswith("unknown message digest algorithm") or \2673 "SHOTGUN_FORCE_CERTIFICATE_VALIDATION" in os.environ:2674 raise2675 if self.config.no_ssl_validation is False:2676 LOG.warning("SSLHandshakeError: this Python installation is incompatible with "2677 "certificates signed with SHA-2. Disabling certificate validation. "2678 "For more information, see http://blog.shotgunsoftware.com/2016/01/"2679 "important-ssl-certificate-renewal-and.html")2680 self._turn_off_ssl_validation()2681 # reload user agent to reflect that we have turned off ssl validation2682 req_headers["user-agent"] = "; ".join(self._user_agents)2683 self._close_connection()2684 if attempt == max_rpc_attempts:2685 raise2686 except Exception:2687 #TODO: LOG ?2688 self._close_connection()2689 if attempt == max_rpc_attempts:2690 raise2691 def _http_request(self, verb, path, body, headers):2692 """2693 Make the actual HTTP request.2694 """2695 url = urlparse.urlunparse((self.config.scheme, self.config.server,2696 path, None, None, None))2697 LOG.debug("Request is %s:%s" % (verb, url))2698 LOG.debug("Request headers are %s" % headers)2699 LOG.debug("Request body is %s" % body)2700 conn = self._get_connection()2701 resp, content = conn.request(url, method=verb, body=body,2702 headers=headers)2703 #http response code is handled else where2704 http_status = (resp.status, resp.reason)2705 resp_headers = dict(2706 (k.lower(), v)2707 for k, v in resp.iteritems()2708 )2709 resp_body = content2710 LOG.debug("Response status is %s %s" % http_status)2711 LOG.debug("Response headers are %s" % resp_headers)2712 LOG.debug("Response body is %s" % resp_body)2713 return (http_status, resp_headers, resp_body)2714 def _parse_http_status(self, status):2715 """2716 Parse the status returned from the http request.2717 :param tuple status: Tuple of (code, reason).2718 :raises: RuntimeError if the http status is non success.2719 """2720 error_code = status[0]2721 errmsg = status[1]2722 if status[0] >= 300:2723 headers = "HTTP error from server"2724 if status[0] == 503:2725 errmsg = "Shotgun is currently down for maintenance or too busy to reply. Please " \2726 "try again later."2727 raise ProtocolError(self.config.server,2728 error_code,2729 errmsg,2730 headers)2731 return2732 def _decode_response(self, headers, body):2733 """2734 Decode the response from the server from the wire format to2735 a python data structure.2736 :param dict headers: Headers from the server.2737 :param str body: Raw response body from the server.2738 :returns: If the content-type starts with application/json or2739 text/javascript the body is json decoded. Otherwise the raw body is2740 returned.2741 :rtype: str2742 """2743 if not body:2744 return body2745 ct = (headers.get("content-type") or "application/json").lower()2746 if ct.startswith("application/json") or ct.startswith("text/javascript"):2747 return self._json_loads(body)2748 return body2749 def _json_loads(self, body):2750 return json.loads(body)2751 def _json_loads_ascii(self, body):2752 """2753 See http://stackoverflow.com/questions/9568672754 """2755 def _decode_list(lst):2756 newlist = []2757 for i in lst:2758 if isinstance(i, unicode):2759 i = i.encode('utf-8')2760 elif isinstance(i, list):2761 i = _decode_list(i)2762 newlist.append(i)2763 return newlist2764 def _decode_dict(dct):2765 newdict = {}2766 for k, v in dct.iteritems():2767 if isinstance(k, unicode):2768 k = k.encode('utf-8')2769 if isinstance(v, unicode):2770 v = v.encode('utf-8')2771 elif isinstance(v, list):2772 v = _decode_list(v)2773 newdict[k] = v2774 return newdict2775 return json.loads(body, object_hook=_decode_dict)2776 def _response_errors(self, sg_response):2777 """2778 Raise any API errors specified in the response.2779 :raises ShotgunError: If the server response contains an exception.2780 """2781 ERR_AUTH = 102 # error code for authentication related problems2782 ERR_2FA = 106 # error code when 2FA authentication is required but no 2FA token provided.2783 ERR_SSO = 108 # error code when SSO is activated on the site, preventing the use of username/password for authentication.2784 if isinstance(sg_response, dict) and sg_response.get("exception"):2785 if sg_response.get("error_code") == ERR_AUTH:2786 raise AuthenticationFault(sg_response.get("message", "Unknown Authentication Error"))2787 elif sg_response.get("error_code") == ERR_2FA:2788 raise MissingTwoFactorAuthenticationFault(sg_response.get("message", "Unknown 2FA Authentication Error"))2789 elif sg_response.get("error_code") == ERR_SSO:2790 raise UserCredentialsNotAllowedForSSOAuthenticationFault(2791 sg_response.get("message", "Authentication using username/password is not allowed for an SSO-enabled Shotgun site")2792 )2793 else:2794 # raise general Fault2795 raise Fault(sg_response.get("message", "Unknown Error"))2796 return2797 def _visit_data(self, data, visitor):2798 """2799 Walk the data (simple python types) and call the visitor.2800 """2801 if not data:2802 return data2803 recursive = self._visit_data2804 if isinstance(data, list):2805 return [recursive(i, visitor) for i in data]2806 if isinstance(data, tuple):2807 return tuple(recursive(i, visitor) for i in data)2808 if isinstance(data, dict):2809 return dict(2810 (k, recursive(v, visitor))2811 for k, v in data.iteritems()2812 )2813 return visitor(data)2814 def _transform_outbound(self, data):2815 """2816 Transform data types or values before they are sent by the client.2817 - changes timezones2818 - converts dates and times to strings2819 """2820 if self.config.convert_datetimes_to_utc:2821 def _change_tz(value):2822 if value.tzinfo is None:2823 value = value.replace(tzinfo=SG_TIMEZONE.local)2824 return value.astimezone(SG_TIMEZONE.utc)2825 else:2826 _change_tz = None2827 local_now = datetime.datetime.now()2828 def _outbound_visitor(value):2829 if isinstance(value, datetime.datetime):2830 if _change_tz:2831 value = _change_tz(value)2832 return value.strftime("%Y-%m-%dT%H:%M:%SZ")2833 if isinstance(value, datetime.date):2834 #existing code did not tz transform dates.2835 return value.strftime("%Y-%m-%d")2836 if isinstance(value, datetime.time):2837 value = local_now.replace(hour=value.hour,2838 minute=value.minute, second=value.second,2839 microsecond=value.microsecond)2840 if _change_tz:2841 value = _change_tz(value)2842 return value.strftime("%Y-%m-%dT%H:%M:%SZ")2843 if isinstance(value, str):2844 # Convert strings to unicode2845 return value.decode("utf-8")2846 return value2847 return self._visit_data(data, _outbound_visitor)2848 def _transform_inbound(self, data):2849 """2850 Transforms data types or values after they are received from the server.2851 """2852 # NOTE: The time zone is removed from the time after it is transformed2853 # to the local time, otherwise it will fail to compare to datetimes2854 # that do not have a time zone.2855 if self.config.convert_datetimes_to_utc:2856 _change_tz = lambda x: x.replace(tzinfo=SG_TIMEZONE.utc)\2857 .astimezone(SG_TIMEZONE.local)2858 else:2859 _change_tz = None2860 def _inbound_visitor(value):2861 if isinstance(value, basestring):2862 if len(value) == 20 and self._DATE_TIME_PATTERN.match(value):2863 try:2864 # strptime was not on datetime in python2.42865 value = datetime.datetime(2866 *time.strptime(value, "%Y-%m-%dT%H:%M:%SZ")[:6])2867 except ValueError:2868 return value2869 if _change_tz:2870 return _change_tz(value)2871 return value2872 return value2873 return self._visit_data(data, _inbound_visitor)2874 # ========================================================================2875 # Connection Functions2876 def _get_connection(self):2877 """2878 Return the current connection or creates a new connection to the current server.2879 """2880 if self._connection is not None:2881 return self._connection2882 if self.config.proxy_server:2883 pi = ProxyInfo(socks.PROXY_TYPE_HTTP, self.config.proxy_server,2884 self.config.proxy_port, proxy_user=self.config.proxy_user,2885 proxy_pass=self.config.proxy_pass)2886 self._connection = Http(timeout=self.config.timeout_secs, ca_certs=self.__ca_certs,2887 proxy_info=pi, disable_ssl_certificate_validation=self.config.no_ssl_validation)2888 else:2889 self._connection = Http(timeout=self.config.timeout_secs, ca_certs=self.__ca_certs,2890 proxy_info=None, disable_ssl_certificate_validation=self.config.no_ssl_validation)2891 return self._connection2892 def _close_connection(self):2893 """2894 Close the current connection.2895 """2896 if self._connection is None:2897 return2898 for conn in self._connection.connections.values():2899 try:2900 conn.close()2901 except Exception:2902 pass2903 self._connection.connections.clear()2904 self._connection = None2905 return2906 # ========================================================================2907 # Utility2908 def _parse_records(self, records):2909 """2910 Parse 'records' returned from the api to do local modifications:2911 - Insert thumbnail urls2912 - Insert local file paths.2913 - Revert &lt; html entities that may be the result of input sanitization2914 mechanisms back to a litteral < character.2915 :param records: List of records (dicts) to process or a single record.2916 :returns: A list of the records processed.2917 """2918 if not records:2919 return []2920 if not isinstance(records, (list, tuple)):2921 records = [records, ]2922 for rec in records:2923 # skip results that aren't entity dictionaries2924 if not isinstance(rec, dict):2925 continue2926 # iterate over each item and check each field for possible injection2927 for k, v in rec.iteritems():2928 if not v:2929 continue2930 # Check for html entities in strings2931 if isinstance(v, types.StringTypes):2932 rec[k] = rec[k].replace('&lt;', '<')2933 # check for thumbnail for older version (<3.3.0) of shotgun2934 if k == 'image' and \2935 self.server_caps.version and \2936 self.server_caps.version < (3, 3, 0):2937 rec['image'] = self._build_thumb_url(rec['type'],2938 rec['id'])2939 continue2940 if isinstance(v, dict) and v.get('link_type') == 'local' \2941 and self.client_caps.local_path_field in v:2942 local_path = v[self.client_caps.local_path_field]2943 v['local_path'] = local_path2944 v['url'] = "file://%s" % (local_path or "",)2945 return records2946 def _build_thumb_url(self, entity_type, entity_id):2947 """2948 Return the URL for the thumbnail of an entity given the entity type and the entity id.2949 Note: This makes a call to the server for every thumbnail.2950 :param entity_type: Entity type the id is for.2951 :param entity_id: id of the entity to get the thumbnail for.2952 :returns: Fully qualified url to the thumbnail.2953 """2954 # Example response from the end point2955 # curl "https://foo.com/upload/get_thumbnail_url?entity_type=Version&entity_id=1"2956 # 12957 # /files/0000/0000/0012/232/shot_thumb.jpg.jpg2958 entity_info = {'e_type':urllib.quote(entity_type),2959 'e_id':urllib.quote(str(entity_id))}2960 url = ("/upload/get_thumbnail_url?" +2961 "entity_type=%(e_type)s&entity_id=%(e_id)s" % entity_info)2962 body = self._make_call("GET", url, None, None)[2]2963 code, thumb_url = body.splitlines()2964 code = int(code)2965 # code of 0 means error, second line is the error code2966 if code == 0:2967 raise ShotgunError(thumb_url)2968 if code == 1:2969 return urlparse.urlunparse((self.config.scheme,2970 self.config.server, thumb_url.strip(), None, None, None))2971 # Comments in prev version said we can get this sometimes.2972 raise RuntimeError("Unknown code %s %s" % (code, thumb_url))2973 def _dict_to_list(self, d, key_name="field_name", value_name="value", extra_data=None):2974 """2975 Utility function to convert a dict into a list dicts using the key_name and value_name keys.2976 e.g. d {'foo' : 'bar'} changed to [{'field_name':'foo', 'value':'bar'}]2977 Any dictionary passed in via extra_data will be merged into the resulting dictionary.2978 e.g. d as above and extra_data of {'foo': {'thing1': 'value1'}} changes into2979 [{'field_name': 'foo', 'value': 'bar', 'thing1': 'value1'}]2980 """2981 ret = []2982 for k, v in (d or {}).iteritems():2983 d = {key_name: k, value_name: v}2984 d.update((extra_data or {}).get(k, {}))2985 ret.append(d)2986 return ret2987 def _dict_to_extra_data(self, d, key_name="value"):2988 """2989 Utility function to convert a dict into a dict compatible with the extra_data arg2990 of _dict_to_list.2991 e.g. d {'foo' : 'bar'} changed to {'foo': {"value": 'bar'}]2992 """2993 return dict([(k, {key_name: v}) for (k,v) in (d or {}).iteritems()])2994 def _upload_file_to_storage(self, path, storage_url):2995 """2996 Internal function to upload an entire file to the Cloud storage.2997 :param str path: Full path to an existing non-empty file on disk to upload.2998 :param str storage_url: Target URL for the uploaded file.2999 """3000 filename = os.path.basename(path)3001 fd = open(path, "rb")3002 try:3003 content_type = mimetypes.guess_type(filename)[0]3004 content_type = content_type or "application/octet-stream"3005 file_size = os.fstat(fd.fileno())[stat.ST_SIZE]3006 self._upload_data_to_storage(fd, content_type, file_size, storage_url )3007 finally:3008 fd.close()3009 LOG.debug("File uploaded to Cloud storage: %s", filename)3010 def _multipart_upload_file_to_storage(self, path, upload_info):3011 """3012 Internal function to upload a file to the Cloud storage in multiple parts.3013 :param str path: Full path to an existing non-empty file on disk to upload.3014 :param dict upload_info: Contains details received from the server, about the upload.3015 """3016 fd = open(path, "rb")3017 try:3018 content_type = mimetypes.guess_type(path)[0]3019 content_type = content_type or "application/octet-stream"3020 file_size = os.fstat(fd.fileno())[stat.ST_SIZE]3021 filename = os.path.basename(path)3022 etags = []3023 part_number = 13024 bytes_read = 03025 chunk_size = self._MULTIPART_UPLOAD_CHUNK_SIZE3026 while bytes_read < file_size:3027 data = fd.read(chunk_size)3028 bytes_read += len(data)3029 part_url = self._get_upload_part_link(upload_info, filename, part_number)3030 etags.append(self._upload_data_to_storage(data, content_type, len(data), part_url ))3031 part_number += 13032 self._complete_multipart_upload(upload_info, filename, etags)3033 finally:3034 fd.close()3035 LOG.debug("File uploaded in multiple parts to Cloud storage: %s", path)3036 def _get_upload_part_link(self, upload_info, filename, part_number):3037 """3038 Internal function to get the url to upload the next part of a file to the3039 Cloud storage, in a multi-part upload process.3040 :param dict upload_info: Contains details received from the server, about the upload.3041 :param str filename: Name of the file for which we want the link.3042 :param int part_number: Part number for the link.3043 :returns: upload url.3044 :rtype: str3045 """3046 params = {3047 "upload_type": upload_info["upload_type"],3048 "filename": filename,3049 "timestamp": upload_info["timestamp"],3050 "upload_id": upload_info["upload_id"],3051 "part_number": part_number3052 }3053 url = urlparse.urlunparse((self.config.scheme, self.config.server,3054 "/upload/api_get_upload_link_for_part", None, None, None))3055 result = self._send_form(url, params)3056 # Response is of the form: 1\n<url> (for success) or 0\n (for failure).3057 # In case of success, we know we the second line of the response contains the3058 # requested URL.3059 if not str(result).startswith("1"):3060 raise ShotgunError("Unable get upload part link: %s" % result)3061 LOG.debug("Got next upload link from server for multipart upload.")3062 return str(result).split("\n")[1]3063 def _upload_data_to_storage(self, data, content_type, size, storage_url):3064 """3065 Internal function to upload data to Cloud storage.3066 :param stream data: Contains details received from the server, about the upload.3067 :param str content_type: Content type of the data stream.3068 :param int size: Number of bytes in the data stream.3069 :param str storage_url: Target URL for the uploaded file.3070 :returns: upload url.3071 :rtype: str3072 """3073 try:3074 opener = urllib2.build_opener(urllib2.HTTPHandler)3075 request = urllib2.Request(storage_url, data=data)3076 request.add_header("Content-Type", content_type)3077 request.add_header("Content-Length", size)3078 request.get_method = lambda: "PUT"3079 result = opener.open(request)3080 etag = result.info().getheader("ETag")3081 except urllib2.HTTPError, e:3082 if e.code == 500:3083 raise ShotgunError("Server encountered an internal error.\n%s\n%s\n\n" % (storage_url, e))3084 else:3085 raise ShotgunError("Unanticipated error occurred uploading to %s: %s" % (storage_url, e))3086 LOG.debug("Part upload completed successfully.")3087 return etag3088 def _complete_multipart_upload(self, upload_info, filename, etags):3089 """3090 Internal function to complete a multi-part upload to the Cloud storage.3091 :param dict upload_info: Contains details received from the server, about the upload.3092 :param str filename: Name of the file for which we want to complete the upload.3093 :param tupple etags: Contains the etag of each uploaded file part.3094 """3095 params = {3096 "upload_type": upload_info["upload_type"],3097 "filename": filename,3098 "timestamp": upload_info["timestamp"],3099 "upload_id": upload_info["upload_id"],3100 "etags": ",".join(etags)3101 }3102 url = urlparse.urlunparse((self.config.scheme, self.config.server,3103 "/upload/api_complete_multipart_upload", None, None, None))3104 result = self._send_form(url, params)3105 # Response is of the form: 1\n or 0\n to indicate success or failure of the call.3106 if not str(result).startswith("1"):3107 raise ShotgunError("Unable get upload part link: %s" % result)3108 def _send_form(self, url, params):3109 """3110 Utility function to send a Form to Shotgun and process any HTTP errors that3111 could occur.3112 :param url: endpoint where the form is sent.3113 :param params: form data3114 :returns: result from the server.3115 """3116 params.update(self._auth_params())3117 opener = self._build_opener(FormPostHandler)3118 # Perform the request3119 try:3120 resp = opener.open(url, params)3121 result = resp.read()3122 # response headers are in str(resp.info()).splitlines()3123 except urllib2.HTTPError, e:3124 if e.code == 500:3125 raise ShotgunError("Server encountered an internal error. "3126 "\n%s\n(%s)\n%s\n\n" % (url, self._sanitize_auth_params(params), e))3127 else:3128 raise ShotgunError("Unanticipated error occurred %s" % (e))3129 return result3130# Helpers from the previous API, left as is.3131# Based on http://code.activestate.com/recipes/146306/3132class FormPostHandler(urllib2.BaseHandler):3133 """3134 Handler for multipart form data3135 """3136 handler_order = urllib2.HTTPHandler.handler_order - 10 # needs to run first3137 def http_request(self, request):3138 data = request.get_data()3139 if data is not None and not isinstance(data, basestring):3140 files = []3141 params = []3142 for key, value in data.items():3143 if isinstance(value, file):3144 files.append((key, value))3145 else:3146 params.append((key, value))3147 if not files:3148 data = urllib.urlencode(params, True) # sequencing on3149 else:3150 boundary, data = self.encode(params, files)3151 content_type = 'multipart/form-data; boundary=%s' % boundary3152 request.add_unredirected_header('Content-Type', content_type)3153 request.add_data(data)3154 return request3155 def encode(self, params, files, boundary=None, buffer=None):3156 if boundary is None:3157 boundary = mimetools.choose_boundary()3158 if buffer is None:3159 buffer = cStringIO.StringIO()3160 for (key, value) in params:3161 buffer.write('--%s\r\n' % boundary)3162 buffer.write('Content-Disposition: form-data; name="%s"' % key)3163 buffer.write('\r\n\r\n%s\r\n' % value)3164 for (key, fd) in files:3165 filename = fd.name.split('/')[-1]3166 content_type = mimetypes.guess_type(filename)[0]3167 content_type = content_type or 'application/octet-stream'3168 file_size = os.fstat(fd.fileno())[stat.ST_SIZE]3169 buffer.write('--%s\r\n' % boundary)3170 c_dis = 'Content-Disposition: form-data; name="%s"; filename="%s"%s'3171 content_disposition = c_dis % (key, filename, '\r\n')3172 buffer.write(content_disposition)3173 buffer.write('Content-Type: %s\r\n' % content_type)3174 buffer.write('Content-Length: %s\r\n' % file_size)3175 fd.seek(0)3176 buffer.write('\r\n%s\r\n' % fd.read())3177 buffer.write('--%s--\r\n\r\n' % boundary)3178 buffer = buffer.getvalue()3179 return boundary, buffer3180 def https_request(self, request):3181 return self.http_request(request)3182def _translate_filters(filters, filter_operator):3183 """3184 Translate filters params into data structure expected by rpc call.3185 """3186 wrapped_filters = {3187 "filter_operator": filter_operator or "all",3188 "filters": filters3189 }3190 return _translate_filters_dict(wrapped_filters)3191def _translate_filters_dict(sg_filter):3192 new_filters = {}3193 filter_operator = sg_filter.get("filter_operator")3194 if filter_operator == "all" or filter_operator == "and":3195 new_filters["logical_operator"] = "and"3196 elif filter_operator == "any" or filter_operator == "or":3197 new_filters["logical_operator"] = "or"3198 else:3199 raise ShotgunError("Invalid filter_operator %s" % filter_operator)3200 if not isinstance(sg_filter["filters"], (list,tuple)):3201 raise ShotgunError("Invalid filters, expected a list or a tuple, got %s"3202 % sg_filter["filters"])3203 new_filters["conditions"] = _translate_filters_list(sg_filter["filters"])3204 return new_filters3205def _translate_filters_list(filters):3206 conditions = []3207 for sg_filter in filters:3208 if isinstance(sg_filter, (list,tuple)):3209 conditions.append(_translate_filters_simple(sg_filter))3210 elif isinstance(sg_filter, dict):3211 conditions.append(_translate_filters_dict(sg_filter))3212 else:3213 raise ShotgunError("Invalid filters, expected a list, tuple or dict, got %s"3214 % sg_filter)3215 return conditions3216def _translate_filters_simple(sg_filter):3217 condition = {3218 "path": sg_filter[0],3219 "relation": sg_filter[1]3220 }3221 values = sg_filter[2:]3222 if len(values) == 1 and isinstance(values[0], (list, tuple)):3223 values = values[0]3224 condition["values"] = values3225 return condition3226def _version_str(version):3227 """3228 Convert a tuple of int's to a '.' separated str.3229 """...

icap_encryption.py

Source:icap_encryption.py

1#!/bin/env python2# -*- coding: utf8 -*-3import datetime4import re5import threading6import os7import subprocess8import getpass9from hashlib import sha25610import hashlib11import base6412import xml.etree.ElementTree as et13try:14 import socketserver15except ImportError:16 import SocketServer17 socketserver = SocketServer18import sys19sys.path.append('.')20from .pyicap import *21from .object_crypto import AESCipher22from .aws_signature_icap import AWSSignature23from .auth_manager import AuthManager24BASE_DIR = os.path.dirname(os.path.dirname(__file__))25class ThreadingSimpleServer(socketserver.ThreadingMixIn, ICAPServer):26 pass27class ICAPHandler(BaseICAPRequestHandler):28 pending_urls = {}29 def request_OPTIONS(self):30 self.set_icap_response(200)31 self.set_icap_header(b'Methods', b'REQMOD')32 self.set_icap_header(b'Service', b'PyICAP Server 1.0')33 self.send_headers(False)34 def request_REQMOD(self):35 """36 fname = re.sub('\/', '_', self.enc_req[1].decode())37 f = open(fname, 'w')38 f.write('Request URL:\n')39 f.write(self.enc_req[1].decode())40 f.write('\n------------\n')41 f.write('Headers:\n')42 for i in self.enc_req_headers:43 f.write(i.decode() + " : ")44 for v in self.enc_req_headers[i]:45 f.write(v.decode() + ',')46 f.write('\n')47 f.close()48 """49 if not self.server.auth_manager.is_setup(): # Authentication manager is not set up50 self.send_error(403, message='error')51 return52 awss = AWSSignature()53 self.set_icap_response(200)54 self.set_enc_request(b' '.join(self.enc_req))55 for h in self.enc_req_headers:56 for v in self.enc_req_headers[h]:57 self.set_enc_header(h, v)58 content = b''59 if not self.has_body:60 if self.enc_req[0] == b'DELETE': # DeleteObject operation61 self.pending_urls[('DELETE', self.enc_req[1].decode())] = ''62 if self.enc_req[0] == b'PUT' and b'x-amz-copy-source' in self.enc_req_headers:63 # Object duplication, copy object key from existing record64 bucket_name = re.search("(?<=^\/)[^\/]+", self.enc_req_headers[b'x-amz-copy-source'][0].decode()).group()65 key = re.search("^.*\.s3.*\.amazonaws.com", self.enc_req[1].decode()).group() + \66 re.search("(?<=^\/{})\/.*".format(bucket_name), self.enc_req_headers[b'x-amz-copy-source'][0].decode()).group()67 key = re.sub("(?<=^https:\/\/)[a-z0-9]+(?=.s3)", bucket_name, key)68 pending_entry = {69 'source': key,70 'new_path': self.enc_req[1].decode()71 }72 self.pending_urls[('PUT', self.enc_req[1].decode())] = pending_entry73 if self.enc_req[0] == b'POST' and b'?uploads' in self.enc_req[1]:74 # New multipart upload (CreateMultipartUpload)75 object_key = re.search("^.*(?=\?)", self.enc_req[1].decode()).group()76 self.pending_urls[('POST', self.enc_req[1].decode())] = {'multipart': 1,77 'object_key': object_key}78 self.send_headers(False)79 return80 if self.preview:81 prevbuf = b''82 while True:83 chunk = self.read_chunk()84 if chunk == b'':85 break86 prevbuf += chunk87 if self.ieof:88 self.send_headers(True)89 if len(prevbuf) > 0:90 self.write_chunk(prevbuf)91 self.write_chunk(b'')92 return93 self.cont()94 self.send_headers(True)95 if len(prevbuf) > 0:96 self.write_chunk(prevbuf)97 while True:98 chunk = self.read_chunk()99 self.write_chunk(chunk)100 if chunk == b'':101 break102 else:103 while True: # Read request body104 chunk = self.read_chunk()105 content += chunk106 if chunk == b'':107 break108 not_object_upload = re.search("\.s3\.(.+)\.amazonaws\.com/\?", self.enc_req[1].decode())109 is_multipart_upload = re.search("^.*\.s3\..*\.amazonaws.com\/.*\?partNumber=", self.enc_req[1].decode()) is not None110 complete_multipart_request = re.search("s3\..*\.amazonaws\.com\/.*\?uploadId=(?!.*partNumber=)",111 self.enc_req[1].decode()) is not None112 if self.enc_req[0] == b'PUT' and not_object_upload is None and not is_multipart_upload:113 # Single object upload114 cp = AESCipher()115 enc_tup = cp.encrypt(content)116 encrypted_content_length = str(len(enc_tup[1])).encode()117 # Update these headers if they exist118 if b'content-md5' in self.enc_headers:119 self.enc_headers.pop(b'content-md5')120 encrypted_content_md5 = hashlib.md5(enc_tup[1]).hexdigest()121 self.set_enc_header(b'content-md5', encrypted_content_md5.encode())122 elif b'x-amz-content-sha256' in self.enc_headers:123 self.enc_headers.pop(b'x-amz-content-sha256')124 encrypted_content_sha256 = sha256(enc_tup[1]).hexdigest()125 self.set_enc_header(b'x-amz-content-sha256', encrypted_content_sha256.encode())126 self.enc_headers.pop(b'content-length')127 self.set_enc_header(b'content-length', encrypted_content_length)128 if b'content-type' in self.enc_headers: # Binary files have no content-type header129 self.enc_headers.pop(b'content-type')130 self.set_enc_header(b'content-type', b'')131 content = enc_tup[1]132 key = enc_tup[0]133 #payload_hash = sha256(enc_tup[1]).hexdigest()134 sig = awss.gen_signature(request=self.enc_req, headers=self.enc_headers) # Generate AWS signature135 self.set_enc_request(b'PUT ' + sig['url'].encode() + b' HTTP/1.1') # Update URL of request136 if b'authorization' in self.enc_headers:137 # Header should always be present if authenticating with header option138 self.enc_headers.pop(b'authorization')139 self.set_enc_header(b'authorization', sig['authorization-header'].encode())140 self.pending_urls[('PUT', sig['url'])] = {'key': key['key'], 'nonce': key['nonce'], 'tag': key['tag']}141 elif self.enc_req[0] == b'PUT' and is_multipart_upload:142 # Multipart upload143 cp = AESCipher()144 object_key = re.search("^.*\.s3\..*\.amazonaws.com\/.*(?=\?)", self.enc_req[1].decode()).group()145 part_num = re.search("(?<=partNumber=)[0-9]+", self.enc_req[1].decode()).group()146 upload_id = re.search("(?<=uploadId=)[^&]+", self.enc_req[1].decode()).group()147 file_params = self.server.auth_manager.get_object(object_key) # Get key from existing record148 params_key = base64.b64decode(file_params.key)149 params_nonce = base64.b64decode(file_params.nonce)150 enc_tup = cp.encrypt(content, key=params_key, nonce=params_nonce)151 content = enc_tup[1]152 part_length = len(enc_tup[1])153 if b'content-md5' in self.enc_headers:154 self.enc_headers.pop(b'content-md5')155 encrypted_content_md5 = base64.b64encode(hashlib.md5(enc_tup[1]).digest())156 self.set_enc_header(b'content-md5', encrypted_content_md5)157 if b'x-amz-content-sha256' in self.enc_headers:158 self.enc_headers.pop(b'x-amz-content-sha256')159 encrypted_content_sha256 = sha256(enc_tup[1]).hexdigest()160 self.set_enc_header(b'x-amz-content-sha256', encrypted_content_sha256.encode())161 self.enc_headers.pop(b'content-length')162 self.set_enc_header(b'content-length', str(part_length).encode())163 if b'content-type' in self.enc_headers: # binary files have no content-type header164 self.enc_headers.pop(b'content-type')165 self.set_enc_header(b'content-type', b'')166 sig = awss.gen_signature(request=self.enc_req, headers=self.enc_headers)167 self.set_enc_request(b'PUT ' + sig['url'].encode() + b' HTTP/1.1')168 if b'authorization' in self.enc_headers:169 self.enc_headers.pop(b'authorization')170 self.set_enc_header(b'authorization', sig['authorization-header'].encode())171 self.pending_urls[('PUT', sig['url'])] = { 'part_num': part_num,172 'part_length': part_length,173 'part_tag': enc_tup[0]['tag'],174 'upload_id': upload_id175 }176 elif self.enc_req[0] == b'POST' and complete_multipart_request is True:177 # Complete multipart upload request178 xmldata = et.fromstring(content.decode())179 valid_parts = xmldata.findall("./{*}Part/{*}PartNumber")180 valid_parts = [x.text for x in valid_parts]181 upload_id = re.search("(?<=\?uploadId=)([^\&]*)", self.enc_req[1].decode()).group()182 all_parts = self.server.auth_manager.get_object_parts(upload_id=upload_id)183 for part in all_parts:184 # Remove invalid parts not referenced by CompleteMultiPartUpload185 if str(part['part_num']) not in valid_parts:186 self.server.auth_manager.delete_part(upload_id=upload_id, part_num=part['part_num'])187 self.send_headers(True)188 self.write_chunk(content)189 def response_OPTIONS(self):190 self.set_icap_response(200)191 self.set_icap_header(b'Methods', b'RESPMOD')192 self.set_icap_header(b'Service', b'PyICAP Server 1.0')193 self.set_icap_header(b'Preview', b'0')194 self.set_icap_header(b'Transfer-Preview', b'*')195 self.set_icap_header(b'Transfer-Ignore', b'jpg,jpeg,gif,png,swf,flv')196 self.set_icap_header(b'Transfer-Complete', b'')197 self.set_icap_header(b'Max-Connections', b'100')198 self.set_icap_header(b'Options-TTL', b'3600')199 self.send_headers(False)200 def read_chunks(self):201 content = b''202 while True:203 chunk = self.read_chunk()204 content += chunk205 if chunk == b'':206 return content207 def response_RESPMOD(self):208 print("resp")209 obj = None210 self.set_icap_response(200)211 self.set_enc_status(b' '.join(self.enc_res_status))212 for h in self.enc_res_headers:213 for v in self.enc_res_headers[h]:214 self.set_enc_header(h, v)215 request_url = self.enc_req[1].decode()216 op = self.enc_req[0].decode()217 url = re.search("^.*\.(s3)\..*(\.amazonaws.com).*(?=\?)", self.enc_req[1].decode()) # Get object key218 if url is None:219 url = re.search("^.*\.(s3)\..*(\.amazonaws.com).*", self.enc_req[1].decode())220 print(self.enc_req)221 print("op: {} url: {}".format(op,url))222 if url is not None:223 url = url.group()224 else:225 url = request_url # No object key226 if (op, request_url) in self.pending_urls.keys():227 obj = self.pending_urls.pop((op, request_url))228 if op == 'PUT' and 'part_num' not in obj: # Response for object upload (regular, not multipart)229 if re.search("\.s3\.(.+)\.amazonaws\.com/\?", request_url) is None:230 if self.enc_res_status[1] == b'200':231 if 'source' in obj: # Upload/copy existing object232 self.server.auth_manager.duplicate_object(object_path=obj['source'],233 new_object_path=obj['new_path'])234 else:235 self.server.auth_manager.add_object(236 object_path=url,237 object_key=obj['key'],238 object_nonce=obj['nonce'],239 object_tag=obj['tag']240 )241 else:242 print("Received response not HTTP 200, will not store key for {}".format(obj['key']))243 elif op == 'PUT' and 'part_num' in obj:244 if self.enc_res_status[1] == b'200': # Part upload succeeded, add it to database245 self.server.auth_manager.add_object_part(upload_id=obj['upload_id'], part_num=obj['part_num'], part_size=obj['part_length'], part_tag=obj['part_tag'])246 elif op == 'DELETE':247 self.server.auth_manager.delete_object(url)248 if not self.has_body:249 self.send_headers(False)250 return251 content = self.read_chunks()252 if self.preview and not self.ieof:253 self.cont()254 content = self.read_chunks()255 if op == 'GET':256 multipart_data = self.server.auth_manager.get_object_parts(url)257 if multipart_data is None:258 # Single object GET, not uploaded in parts259 obj_entry = self.server.auth_manager.get_object(object_path=url) # TODO: merge ops in single&multipart uploaded260 if obj_entry is not None: # GET response for known object261 cp = AESCipher()262 json_key = {'key': obj_entry.key, 'nonce': obj_entry.nonce, 'tag': obj_entry.tag}263 content = cp.decrypt(json_key, content)264 else: # No object found in database265 pass266 elif multipart_data is not None:267 # GET object uploaded in parts268 decrypted_content = b''269 obj_entry = self.server.auth_manager.get_object(object_path=url)270 index = 0271 for item in multipart_data: # Decrypt each offset from database values272 cp = AESCipher()273 json_key = {'key': obj_entry.key, 'nonce': obj_entry.nonce, 'tag': item['part_tag']}274 piece = content[index:index + item['part_size']]275 decrypted_piece = cp.decrypt(json_key, piece)276 decrypted_content += decrypted_piece277 index += item['part_size']278 content = decrypted_content279 if op == 'POST':280 if re.search("\.s3\.(.+)\.amazonaws\.com/\?delete$", request_url) is not None:281 # Bucket paths starting with /? (no file key) are always non-upload config requests282 xmldata = et.fromstring(content.decode())283 for item in xmldata.findall("./{*}Deleted/{*}Key"):284 key = request_url[:-7] + item.text285 self.server.auth_manager.delete_object(key)286 elif obj is not None and 'multipart' in obj: # Confirmed multipart upload initiated287 xmldata = et.fromstring(content.decode())288 upload_id = xmldata.find("./{*}UploadId").text289 self.server.auth_manager.new_multipart_upload(obj['object_key'], upload_id)290 self.send_headers(True)291 self.write_chunk(content)292 self.write_chunk(b'')293class S3_encryption_icap(object):294 def __init__(self, port, auth_manager):295 if port < 1024 or port > 65535:296 self.port = 1344297 else:298 self.port = port299 self.server = ThreadingSimpleServer((b'', self.port), ICAPHandler)300 self.server.auth_manager = auth_manager301 self.run = False302 def start_server(self):303 t = threading.current_thread()304 self.reload_squid()305 # Start handle request loop306 while getattr(t, "run", False):307 self.server.handle_request()308 print("stopped " + str(getattr(t, "run", "def")))309 def stop_server(self):310 self.server.shutdown()311 self.run = False312 def reload_squid(self): #Reload Squid process using system script313 script_path = os.path.join(BASE_DIR, "app/squid_reload.sh")314 try:315 subprocess.run(["sudo", script_path], check=True, capture_output=True)316 except subprocess.CalledProcessError as e:317 print(e)318 print("Could not reload squid. Make sure user {} has non-interactive execute permission for {}".format(getpass.getuser(), script_path))319 return e.stderr...

Automation Testing Tutorials

Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.

LambdaTest Learning Hubs:

• JUnit Tutorial
• TestNG Tutorial
• Webdriver Tutorial
• WebDriverIO Tutorial
• Protractor Tutorial
• Selenium 4 Tutorial
• Jenkins Tutorial
• NUnit Tutorial
• Jest Tutorial
• Playwright Tutorial
• Cypress Tutorial
• PyTest Tutorial

YouTube

You could also refer to video tutorials over LambdaTest YouTube channel to get step by step demonstration from industry experts.

Run localstack automation tests on LambdaTest cloud grid

Perform automation testing on 3000+ real desktop and mobile devices online.

View sample repo View DocumentationSign up Free

›

_