KaneAI - World’s First E2E Software Testing Agent.Book a Demo Join TestMu Conference

How to use resolution method in ATX

Best Python code snippet using ATX

resolutionCache.ts

Source:resolutionCache.ts

1/*@internal*/2namespace ts {3    /** This is the cache of module/typedirectives resolution that can be retained across program */4    export interface ResolutionCache {5        startRecordingFilesWithChangedResolutions(): void;6        finishRecordingFilesWithChangedResolutions(): Path[] | undefined;78        resolveModuleNames(moduleNames: string[], containingFile: string, reusedNames: string[] | undefined, redirectedReference?: ResolvedProjectReference): (ResolvedModuleFull | undefined)[];9        getResolvedModuleWithFailedLookupLocationsFromCache(moduleName: string, containingFile: string): CachedResolvedModuleWithFailedLookupLocations | undefined;10        resolveTypeReferenceDirectives(typeDirectiveNames: string[], containingFile: string, redirectedReference?: ResolvedProjectReference): (ResolvedTypeReferenceDirective | undefined)[];1112        invalidateResolutionOfFile(filePath: Path): void;13        removeResolutionsOfFile(filePath: Path): void;14        removeResolutionsFromProjectReferenceRedirects(filePath: Path): void;15        setFilesWithInvalidatedNonRelativeUnresolvedImports(filesWithUnresolvedImports: Map<ReadonlyArray<string>>): void;16        createHasInvalidatedResolution(forceAllFilesAsInvalidated?: boolean): HasInvalidatedResolution;1718        startCachingPerDirectoryResolution(): void;19        finishCachingPerDirectoryResolution(): void;2021        updateTypeRootsWatch(): void;22        closeTypeRootsWatch(): void;2324        clear(): void;25    }2627    interface ResolutionWithFailedLookupLocations {28        readonly failedLookupLocations: ReadonlyArray<string>;29        isInvalidated?: boolean;30        refCount?: number;31    }3233    interface ResolutionWithResolvedFileName {34        resolvedFileName: string | undefined;35    }3637    interface CachedResolvedModuleWithFailedLookupLocations extends ResolvedModuleWithFailedLookupLocations, ResolutionWithFailedLookupLocations {38    }3940    interface CachedResolvedTypeReferenceDirectiveWithFailedLookupLocations extends ResolvedTypeReferenceDirectiveWithFailedLookupLocations, ResolutionWithFailedLookupLocations {41    }4243    export interface ResolutionCacheHost extends ModuleResolutionHost {44        toPath(fileName: string): Path;45        getCanonicalFileName: GetCanonicalFileName;46        getCompilationSettings(): CompilerOptions;47        watchDirectoryOfFailedLookupLocation(directory: string, cb: DirectoryWatcherCallback, flags: WatchDirectoryFlags): FileWatcher;48        onInvalidatedResolution(): void;49        watchTypeRootsDirectory(directory: string, cb: DirectoryWatcherCallback, flags: WatchDirectoryFlags): FileWatcher;50        onChangedAutomaticTypeDirectiveNames(): void;51        getCachedDirectoryStructureHost(): CachedDirectoryStructureHost | undefined;52        projectName?: string;53        getGlobalCache?(): string | undefined;54        writeLog(s: string): void;55        maxNumberOfFilesToIterateForInvalidation?: number;56        getCurrentProgram(): Program | undefined;57    }5859    interface DirectoryWatchesOfFailedLookup {60        /** watcher for the directory of failed lookup */61        watcher: FileWatcher;62        /** ref count keeping this directory watch alive */63        refCount: number;64        /** is the directory watched being non recursive */65        nonRecursive?: boolean;66    }6768    interface DirectoryOfFailedLookupWatch {69        dir: string;70        dirPath: Path;71        nonRecursive?: boolean;72    }7374    export function isPathInNodeModulesStartingWithDot(path: Path) {75        return stringContains(path, "/node_modules/.");76    }7778    export const maxNumberOfFilesToIterateForInvalidation = 256;7980    type GetResolutionWithResolvedFileName<T extends ResolutionWithFailedLookupLocations = ResolutionWithFailedLookupLocations, R extends ResolutionWithResolvedFileName = ResolutionWithResolvedFileName> =81        (resolution: T) => R | undefined;8283    export function createResolutionCache(resolutionHost: ResolutionCacheHost, rootDirForResolution: string | undefined, logChangesWhenResolvingModule: boolean): ResolutionCache {84        let filesWithChangedSetOfUnresolvedImports: Path[] | undefined;85        let filesWithInvalidatedResolutions: Map<true> | undefined;86        let filesWithInvalidatedNonRelativeUnresolvedImports: ReadonlyMap<ReadonlyArray<string>> | undefined;87        let allFilesHaveInvalidatedResolution = false;88        const nonRelativeExternalModuleResolutions = createMultiMap<ResolutionWithFailedLookupLocations>();8990        const getCurrentDirectory = memoize(() => resolutionHost.getCurrentDirectory!()); // TODO: GH#1821791        const cachedDirectoryStructureHost = resolutionHost.getCachedDirectoryStructureHost();9293        // The resolvedModuleNames and resolvedTypeReferenceDirectives are the cache of resolutions per file.94        // The key in the map is source file's path.95        // The values are Map of resolutions with key being name lookedup.96        const resolvedModuleNames = createMap<Map<CachedResolvedModuleWithFailedLookupLocations>>();97        const perDirectoryResolvedModuleNames: CacheWithRedirects<Map<CachedResolvedModuleWithFailedLookupLocations>> = createCacheWithRedirects();98        const nonRelativeModuleNameCache: CacheWithRedirects<PerModuleNameCache> = createCacheWithRedirects();99        const moduleResolutionCache = createModuleResolutionCacheWithMaps(100            perDirectoryResolvedModuleNames,101            nonRelativeModuleNameCache,102            getCurrentDirectory(),103            resolutionHost.getCanonicalFileName104        );105106        const resolvedTypeReferenceDirectives = createMap<Map<CachedResolvedTypeReferenceDirectiveWithFailedLookupLocations>>();107        const perDirectoryResolvedTypeReferenceDirectives: CacheWithRedirects<Map<CachedResolvedTypeReferenceDirectiveWithFailedLookupLocations>> = createCacheWithRedirects();108109        /**110         * These are the extensions that failed lookup files will have by default,111         * any other extension of failed lookup will be store that path in custom failed lookup path112         * This helps in not having to comb through all resolutions when files are added/removed113         * Note that .d.ts file also has .d.ts extension hence will be part of default extensions114         */115        const failedLookupDefaultExtensions = [Extension.Ts, Extension.Tsx, Extension.Js, Extension.Jsx, Extension.Json];116        const customFailedLookupPaths = createMap<number>();117118        const directoryWatchesOfFailedLookups = createMap<DirectoryWatchesOfFailedLookup>();119        const rootDir = rootDirForResolution && removeTrailingDirectorySeparator(getNormalizedAbsolutePath(rootDirForResolution, getCurrentDirectory()));120        const rootPath = (rootDir && resolutionHost.toPath(rootDir)) as Path; // TODO: GH#18217121122        // TypeRoot watches for the types that get added as part of getAutomaticTypeDirectiveNames123        const typeRootsWatches = createMap<FileWatcher>();124125        return {126            startRecordingFilesWithChangedResolutions,127            finishRecordingFilesWithChangedResolutions,128            // perDirectoryResolvedModuleNames and perDirectoryResolvedTypeReferenceDirectives could be non empty if there was exception during program update129            // (between startCachingPerDirectoryResolution and finishCachingPerDirectoryResolution)130            startCachingPerDirectoryResolution: clearPerDirectoryResolutions,131            finishCachingPerDirectoryResolution,132            resolveModuleNames,133            getResolvedModuleWithFailedLookupLocationsFromCache,134            resolveTypeReferenceDirectives,135            removeResolutionsFromProjectReferenceRedirects,136            removeResolutionsOfFile,137            invalidateResolutionOfFile,138            setFilesWithInvalidatedNonRelativeUnresolvedImports,139            createHasInvalidatedResolution,140            updateTypeRootsWatch,141            closeTypeRootsWatch,142            clear143        };144145        function getResolvedModule(resolution: CachedResolvedModuleWithFailedLookupLocations) {146            return resolution.resolvedModule;147        }148149        function getResolvedTypeReferenceDirective(resolution: CachedResolvedTypeReferenceDirectiveWithFailedLookupLocations) {150            return resolution.resolvedTypeReferenceDirective;151        }152153        function isInDirectoryPath(dir: Path | undefined, file: Path) {154            if (dir === undefined || file.length <= dir.length) {155                return false;156            }157            return startsWith(file, dir) && file[dir.length] === directorySeparator;158        }159160        function clear() {161            clearMap(directoryWatchesOfFailedLookups, closeFileWatcherOf);162            customFailedLookupPaths.clear();163            nonRelativeExternalModuleResolutions.clear();164            closeTypeRootsWatch();165            resolvedModuleNames.clear();166            resolvedTypeReferenceDirectives.clear();167            allFilesHaveInvalidatedResolution = false;168            // perDirectoryResolvedModuleNames and perDirectoryResolvedTypeReferenceDirectives could be non empty if there was exception during program update169            // (between startCachingPerDirectoryResolution and finishCachingPerDirectoryResolution)170            clearPerDirectoryResolutions();171        }172173        function startRecordingFilesWithChangedResolutions() {174            filesWithChangedSetOfUnresolvedImports = [];175        }176177        function finishRecordingFilesWithChangedResolutions() {178            const collected = filesWithChangedSetOfUnresolvedImports;179            filesWithChangedSetOfUnresolvedImports = undefined;180            return collected;181        }182183        function isFileWithInvalidatedNonRelativeUnresolvedImports(path: Path): boolean {184            if (!filesWithInvalidatedNonRelativeUnresolvedImports) {185                return false;186            }187188            // Invalidated if file has unresolved imports189            const value = filesWithInvalidatedNonRelativeUnresolvedImports.get(path);190            return !!value && !!value.length;191        }192193        function createHasInvalidatedResolution(forceAllFilesAsInvalidated?: boolean): HasInvalidatedResolution {194            if (allFilesHaveInvalidatedResolution || forceAllFilesAsInvalidated) {195                // Any file asked would have invalidated resolution196                filesWithInvalidatedResolutions = undefined;197                return returnTrue;198            }199            const collected = filesWithInvalidatedResolutions;200            filesWithInvalidatedResolutions = undefined;201            return path => (!!collected && collected.has(path)) ||202                isFileWithInvalidatedNonRelativeUnresolvedImports(path);203        }204205        function clearPerDirectoryResolutions() {206            perDirectoryResolvedModuleNames.clear();207            nonRelativeModuleNameCache.clear();208            perDirectoryResolvedTypeReferenceDirectives.clear();209            nonRelativeExternalModuleResolutions.forEach(watchFailedLookupLocationOfNonRelativeModuleResolutions);210            nonRelativeExternalModuleResolutions.clear();211        }212213        function finishCachingPerDirectoryResolution() {214            allFilesHaveInvalidatedResolution = false;215            filesWithInvalidatedNonRelativeUnresolvedImports = undefined;216            clearPerDirectoryResolutions();217            directoryWatchesOfFailedLookups.forEach((watcher, path) => {218                if (watcher.refCount === 0) {219                    directoryWatchesOfFailedLookups.delete(path);220                    watcher.watcher.close();221                }222            });223        }224225        function resolveModuleName(moduleName: string, containingFile: string, compilerOptions: CompilerOptions, host: ModuleResolutionHost, redirectedReference?: ResolvedProjectReference): CachedResolvedModuleWithFailedLookupLocations {226            const primaryResult = ts.resolveModuleName(moduleName, containingFile, compilerOptions, host, moduleResolutionCache, redirectedReference);227            // return result immediately only if global cache support is not enabled or if it is .ts, .tsx or .d.ts228            if (!resolutionHost.getGlobalCache) {229                return primaryResult;230            }231232            // otherwise try to load typings from @types233            const globalCache = resolutionHost.getGlobalCache();234            if (globalCache !== undefined && !isExternalModuleNameRelative(moduleName) && !(primaryResult.resolvedModule && extensionIsTS(primaryResult.resolvedModule.extension))) {235                // create different collection of failed lookup locations for second pass236                // if it will fail and we've already found something during the first pass - we don't want to pollute its results237                const { resolvedModule, failedLookupLocations } = loadModuleFromGlobalCache(moduleName, resolutionHost.projectName, compilerOptions, host, globalCache);238                if (resolvedModule) {239                    return { resolvedModule, failedLookupLocations: addRange(primaryResult.failedLookupLocations as string[], failedLookupLocations) };240                }241            }242243            // Default return the result from the first pass244            return primaryResult;245        }246247        function resolveNamesWithLocalCache<T extends ResolutionWithFailedLookupLocations, R extends ResolutionWithResolvedFileName>(248            names: ReadonlyArray<string>,249            containingFile: string,250            redirectedReference: ResolvedProjectReference | undefined,251            cache: Map<Map<T>>,252            perDirectoryCacheWithRedirects: CacheWithRedirects<Map<T>>,253            loader: (name: string, containingFile: string, options: CompilerOptions, host: ModuleResolutionHost, redirectedReference?: ResolvedProjectReference) => T,254            getResolutionWithResolvedFileName: GetResolutionWithResolvedFileName<T, R>,255            shouldRetryResolution: (t: T) => boolean,256            reusedNames: ReadonlyArray<string> | undefined,257            logChanges: boolean): (R | undefined)[] {258259            const path = resolutionHost.toPath(containingFile);260            const resolutionsInFile = cache.get(path) || cache.set(path, createMap()).get(path)!;261            const dirPath = getDirectoryPath(path);262            const perDirectoryCache = perDirectoryCacheWithRedirects.getOrCreateMapOfCacheRedirects(redirectedReference);263            let perDirectoryResolution = perDirectoryCache.get(dirPath);264            if (!perDirectoryResolution) {265                perDirectoryResolution = createMap();266                perDirectoryCache.set(dirPath, perDirectoryResolution);267            }268            const resolvedModules: (R | undefined)[] = [];269            const compilerOptions = resolutionHost.getCompilationSettings();270            const hasInvalidatedNonRelativeUnresolvedImport = logChanges && isFileWithInvalidatedNonRelativeUnresolvedImports(path);271272            // All the resolutions in this file are invalidated if this file wasnt resolved using same redirect273            const program = resolutionHost.getCurrentProgram();274            const oldRedirect = program && program.getResolvedProjectReferenceToRedirect(containingFile);275            const unmatchedRedirects = oldRedirect ?276                !redirectedReference || redirectedReference.sourceFile.path !== oldRedirect.sourceFile.path :277                !!redirectedReference;278279            const seenNamesInFile = createMap<true>();280            for (const name of names) {281                let resolution = resolutionsInFile.get(name);282                // Resolution is valid if it is present and not invalidated283                if (!seenNamesInFile.has(name) &&284                    allFilesHaveInvalidatedResolution || unmatchedRedirects || !resolution || resolution.isInvalidated ||285                    // If the name is unresolved import that was invalidated, recalculate286                    (hasInvalidatedNonRelativeUnresolvedImport && !isExternalModuleNameRelative(name) && shouldRetryResolution(resolution))) {287                    const existingResolution = resolution;288                    const resolutionInDirectory = perDirectoryResolution.get(name);289                    if (resolutionInDirectory) {290                        resolution = resolutionInDirectory;291                    }292                    else {293                        resolution = loader(name, containingFile, compilerOptions, resolutionHost, redirectedReference);294                        perDirectoryResolution.set(name, resolution);295                    }296                    resolutionsInFile.set(name, resolution);297                    watchFailedLookupLocationsOfExternalModuleResolutions(name, resolution);298                    if (existingResolution) {299                        stopWatchFailedLookupLocationOfResolution(existingResolution);300                    }301302                    if (logChanges && filesWithChangedSetOfUnresolvedImports && !resolutionIsEqualTo(existingResolution, resolution)) {303                        filesWithChangedSetOfUnresolvedImports.push(path);304                        // reset log changes to avoid recording the same file multiple times305                        logChanges = false;306                    }307                }308                Debug.assert(resolution !== undefined && !resolution.isInvalidated);309                seenNamesInFile.set(name, true);310                resolvedModules.push(getResolutionWithResolvedFileName(resolution));311            }312313            // Stop watching and remove the unused name314            resolutionsInFile.forEach((resolution, name) => {315                if (!seenNamesInFile.has(name) && !contains(reusedNames, name)) {316                    stopWatchFailedLookupLocationOfResolution(resolution);317                    resolutionsInFile.delete(name);318                }319            });320321            return resolvedModules;322323            function resolutionIsEqualTo(oldResolution: T | undefined, newResolution: T | undefined): boolean {324                if (oldResolution === newResolution) {325                    return true;326                }327                if (!oldResolution || !newResolution) {328                    return false;329                }330                const oldResult = getResolutionWithResolvedFileName(oldResolution);331                const newResult = getResolutionWithResolvedFileName(newResolution);332                if (oldResult === newResult) {333                    return true;334                }335                if (!oldResult || !newResult) {336                    return false;337                }338                return oldResult.resolvedFileName === newResult.resolvedFileName;339            }340        }341342        function resolveTypeReferenceDirectives(typeDirectiveNames: string[], containingFile: string, redirectedReference?: ResolvedProjectReference): (ResolvedTypeReferenceDirective | undefined)[] {343            return resolveNamesWithLocalCache<CachedResolvedTypeReferenceDirectiveWithFailedLookupLocations, ResolvedTypeReferenceDirective>(344                typeDirectiveNames, containingFile, redirectedReference,345                resolvedTypeReferenceDirectives, perDirectoryResolvedTypeReferenceDirectives,346                resolveTypeReferenceDirective, getResolvedTypeReferenceDirective,347                /*shouldRetryResolution*/ resolution => resolution.resolvedTypeReferenceDirective === undefined,348                /*reusedNames*/ undefined, /*logChanges*/ false349            );350        }351352        function resolveModuleNames(moduleNames: string[], containingFile: string, reusedNames: string[] | undefined, redirectedReference?: ResolvedProjectReference): (ResolvedModuleFull | undefined)[] {353            return resolveNamesWithLocalCache<CachedResolvedModuleWithFailedLookupLocations, ResolvedModuleFull>(354                moduleNames, containingFile, redirectedReference,355                resolvedModuleNames, perDirectoryResolvedModuleNames,356                resolveModuleName, getResolvedModule,357                /*shouldRetryResolution*/ resolution => !resolution.resolvedModule || !resolutionExtensionIsTSOrJson(resolution.resolvedModule.extension),358                reusedNames, logChangesWhenResolvingModule359            );360        }361362        function getResolvedModuleWithFailedLookupLocationsFromCache(moduleName: string, containingFile: string): CachedResolvedModuleWithFailedLookupLocations | undefined {363            const cache = resolvedModuleNames.get(resolutionHost.toPath(containingFile));364            return cache && cache.get(moduleName);365        }366367        function isNodeModulesDirectory(dirPath: Path) {368            return endsWith(dirPath, "/node_modules");369        }370371        function isNodeModulesAtTypesDirectory(dirPath: Path) {372            return endsWith(dirPath, "/node_modules/@types");373        }374375        /**376         * Filter out paths like377         * "/", "/user", "/user/username", "/user/username/folderAtRoot",378         * "c:/", "c:/users", "c:/users/username", "c:/users/username/folderAtRoot", "c:/folderAtRoot"379         * @param dirPath380         */381        function canWatchDirectory(dirPath: Path) {382            const rootLength = getRootLength(dirPath);383            if (dirPath.length === rootLength) {384                // Ignore "/", "c:/"385                return false;386            }387388            const nextDirectorySeparator = dirPath.indexOf(directorySeparator, rootLength);389            if (nextDirectorySeparator === -1) {390                // ignore "/user", "c:/users" or "c:/folderAtRoot"391                return false;392            }393394            if (dirPath.charCodeAt(0) !== CharacterCodes.slash &&395                dirPath.substr(rootLength, nextDirectorySeparator).search(/users/i) === -1) {396                // Paths like c:/folderAtRoot/subFolder are allowed397                return true;398            }399400            for (let searchIndex = nextDirectorySeparator + 1, searchLevels = 2; searchLevels > 0; searchLevels--) {401                searchIndex = dirPath.indexOf(directorySeparator, searchIndex) + 1;402                if (searchIndex === 0) {403                    // Folder isnt at expected minimun levels404                    return false;405                }406            }407            return true;408        }409410        function getDirectoryToWatchFailedLookupLocation(failedLookupLocation: string, failedLookupLocationPath: Path): DirectoryOfFailedLookupWatch | undefined {411            if (isInDirectoryPath(rootPath, failedLookupLocationPath)) {412                // Ensure failed look up is normalized path413                failedLookupLocation = isRootedDiskPath(failedLookupLocation) ? normalizePath(failedLookupLocation) : getNormalizedAbsolutePath(failedLookupLocation, getCurrentDirectory());414                Debug.assert(failedLookupLocation.length === failedLookupLocationPath.length, `FailedLookup: ${failedLookupLocation} failedLookupLocationPath: ${failedLookupLocationPath}`); // tslint:disable-line415                const subDirectoryInRoot = failedLookupLocationPath.indexOf(directorySeparator, rootPath.length + 1);416                if (subDirectoryInRoot !== -1) {417                    // Instead of watching root, watch directory in root to avoid watching excluded directories not needed for module resolution418                    return { dir: failedLookupLocation.substr(0, subDirectoryInRoot), dirPath: failedLookupLocationPath.substr(0, subDirectoryInRoot) as Path };419                }420                else {421                    // Always watch root directory non recursively422                    return { dir: rootDir!, dirPath: rootPath, nonRecursive: false }; // TODO: GH#18217423                }424            }425426            return getDirectoryToWatchFromFailedLookupLocationDirectory(427                getDirectoryPath(getNormalizedAbsolutePath(failedLookupLocation, getCurrentDirectory())),428                getDirectoryPath(failedLookupLocationPath)429            );430        }431432        function getDirectoryToWatchFromFailedLookupLocationDirectory(dir: string, dirPath: Path): DirectoryOfFailedLookupWatch | undefined {433            // If directory path contains node module, get the most parent node_modules directory for watching434            while (pathContainsNodeModules(dirPath)) {435                dir = getDirectoryPath(dir);436                dirPath = getDirectoryPath(dirPath);437            }438439            // If the directory is node_modules use it to watch, always watch it recursively440            if (isNodeModulesDirectory(dirPath)) {441                return canWatchDirectory(getDirectoryPath(dirPath)) ? { dir, dirPath } : undefined;442            }443444            let nonRecursive = true;445            // Use some ancestor of the root directory446            let subDirectoryPath: Path | undefined, subDirectory: string | undefined;447            if (rootPath !== undefined) {448                while (!isInDirectoryPath(dirPath, rootPath)) {449                    const parentPath = getDirectoryPath(dirPath);450                    if (parentPath === dirPath) {451                        break;452                    }453                    nonRecursive = false;454                    subDirectoryPath = dirPath;455                    subDirectory = dir;456                    dirPath = parentPath;457                    dir = getDirectoryPath(dir);458                }459            }460461            return canWatchDirectory(dirPath) ? { dir: subDirectory || dir, dirPath: subDirectoryPath || dirPath, nonRecursive } : undefined;462        }463464        function isPathWithDefaultFailedLookupExtension(path: Path) {465            return fileExtensionIsOneOf(path, failedLookupDefaultExtensions);466        }467468        function watchFailedLookupLocationsOfExternalModuleResolutions(name: string, resolution: ResolutionWithFailedLookupLocations) {469            // No need to set the resolution refCount470            if (resolution.failedLookupLocations && resolution.failedLookupLocations.length) {471                if (resolution.refCount) {472                    resolution.refCount++;473                }474                else {475                    resolution.refCount = 1;476                    if (isExternalModuleNameRelative(name)) {477                        watchFailedLookupLocationOfResolution(resolution);478                    }479                    else {480                        nonRelativeExternalModuleResolutions.add(name, resolution);481                    }482                }483            }484        }485486        function watchFailedLookupLocationOfResolution(resolution: ResolutionWithFailedLookupLocations) {487            Debug.assert(!!resolution.refCount);488489            const { failedLookupLocations } = resolution;490            let setAtRoot = false;491            for (const failedLookupLocation of failedLookupLocations) {492                const failedLookupLocationPath = resolutionHost.toPath(failedLookupLocation);493                const toWatch = getDirectoryToWatchFailedLookupLocation(failedLookupLocation, failedLookupLocationPath);494                if (toWatch) {495                    const { dir, dirPath, nonRecursive } = toWatch;496                    // If the failed lookup location path is not one of the supported extensions,497                    // store it in the custom path498                    if (!isPathWithDefaultFailedLookupExtension(failedLookupLocationPath)) {499                        const refCount = customFailedLookupPaths.get(failedLookupLocationPath) || 0;500                        customFailedLookupPaths.set(failedLookupLocationPath, refCount + 1);501                    }502                    if (dirPath === rootPath) {503                        Debug.assert(!nonRecursive);504                        setAtRoot = true;505                    }506                    else {507                        setDirectoryWatcher(dir, dirPath, nonRecursive);508                    }509                }510            }511512            if (setAtRoot) {513                // This is always non recursive514                setDirectoryWatcher(rootDir!, rootPath, /*nonRecursive*/ true); // TODO: GH#18217515            }516        }517518        function setRefCountToUndefined(resolution: ResolutionWithFailedLookupLocations) {519            resolution.refCount = undefined;520        }521522        function watchFailedLookupLocationOfNonRelativeModuleResolutions(resolutions: ResolutionWithFailedLookupLocations[], name: string) {523            const program = resolutionHost.getCurrentProgram();524            const updateResolution = program && program.getTypeChecker().tryFindAmbientModuleWithoutAugmentations(name) ?525                setRefCountToUndefined : watchFailedLookupLocationOfResolution;526            resolutions.forEach(updateResolution);527        }528529        function setDirectoryWatcher(dir: string, dirPath: Path, nonRecursive?: boolean) {530            const dirWatcher = directoryWatchesOfFailedLookups.get(dirPath);531            if (dirWatcher) {532                Debug.assert(!!nonRecursive === !!dirWatcher.nonRecursive);533                dirWatcher.refCount++;534            }535            else {536                directoryWatchesOfFailedLookups.set(dirPath, { watcher: createDirectoryWatcher(dir, dirPath, nonRecursive), refCount: 1, nonRecursive });537            }538        }539540        function stopWatchFailedLookupLocationOfResolution(resolution: ResolutionWithFailedLookupLocations) {541            if (!resolution.refCount) {542                return;543            }544545            resolution.refCount--;546            if (resolution.refCount) {547                return;548            }549550            const { failedLookupLocations } = resolution;551            let removeAtRoot = false;552            for (const failedLookupLocation of failedLookupLocations) {553                const failedLookupLocationPath = resolutionHost.toPath(failedLookupLocation);554                const toWatch = getDirectoryToWatchFailedLookupLocation(failedLookupLocation, failedLookupLocationPath);555                if (toWatch) {556                    const { dirPath } = toWatch;557                    const refCount = customFailedLookupPaths.get(failedLookupLocationPath);558                    if (refCount) {559                        if (refCount === 1) {560                            customFailedLookupPaths.delete(failedLookupLocationPath);561                        }562                        else {563                            Debug.assert(refCount > 1);564                            customFailedLookupPaths.set(failedLookupLocationPath, refCount - 1);565                        }566                    }567568                    if (dirPath === rootPath) {569                        removeAtRoot = true;570                    }571                    else {572                        removeDirectoryWatcher(dirPath);573                    }574                }575            }576            if (removeAtRoot) {577                removeDirectoryWatcher(rootPath);578            }579        }580581        function removeDirectoryWatcher(dirPath: string) {582            const dirWatcher = directoryWatchesOfFailedLookups.get(dirPath)!;583            // Do not close the watcher yet since it might be needed by other failed lookup locations.584            dirWatcher.refCount--;585        }586587        function createDirectoryWatcher(directory: string, dirPath: Path, nonRecursive: boolean | undefined) {588            return resolutionHost.watchDirectoryOfFailedLookupLocation(directory, fileOrDirectory => {589                const fileOrDirectoryPath = resolutionHost.toPath(fileOrDirectory);590                if (cachedDirectoryStructureHost) {591                    // Since the file existence changed, update the sourceFiles cache592                    cachedDirectoryStructureHost.addOrDeleteFileOrDirectory(fileOrDirectory, fileOrDirectoryPath);593                }594595                if (!allFilesHaveInvalidatedResolution && invalidateResolutionOfFailedLookupLocation(fileOrDirectoryPath, dirPath === fileOrDirectoryPath)) {596                    resolutionHost.onInvalidatedResolution();597                }598            }, nonRecursive ? WatchDirectoryFlags.None : WatchDirectoryFlags.Recursive);599        }600601        function removeResolutionsOfFileFromCache(cache: Map<Map<ResolutionWithFailedLookupLocations>>, filePath: Path) {602            // Deleted file, stop watching failed lookups for all the resolutions in the file603            const resolutions = cache.get(filePath);604            if (resolutions) {605                resolutions.forEach(stopWatchFailedLookupLocationOfResolution);606                cache.delete(filePath);607            }608        }609610        function removeResolutionsFromProjectReferenceRedirects(filePath: Path) {611            if (!fileExtensionIs(filePath, Extension.Json)) { return; }612613            const program = resolutionHost.getCurrentProgram();614            if (!program) { return; }615616            // If this file is input file for the referenced project, get it617            const resolvedProjectReference = program.getResolvedProjectReferenceByPath(filePath);618            if (!resolvedProjectReference) { return; }619620            // filePath is for the projectReference and the containing file is from this project reference, invalidate the resolution621            resolvedProjectReference.commandLine.fileNames.forEach(f => removeResolutionsOfFile(resolutionHost.toPath(f)));622        }623624        function removeResolutionsOfFile(filePath: Path) {625            removeResolutionsOfFileFromCache(resolvedModuleNames, filePath);626            removeResolutionsOfFileFromCache(resolvedTypeReferenceDirectives, filePath);627        }628629        function invalidateResolutionCache<T extends ResolutionWithFailedLookupLocations, R extends ResolutionWithResolvedFileName>(630            cache: Map<Map<T>>,631            isInvalidatedResolution: (resolution: T, getResolutionWithResolvedFileName: GetResolutionWithResolvedFileName<T, R>) => boolean,632            getResolutionWithResolvedFileName: GetResolutionWithResolvedFileName<T, R>633        ) {634            const seen = createMap<Map<true>>();635            cache.forEach((resolutions, containingFilePath) => {636                const dirPath = getDirectoryPath(containingFilePath);637                let seenInDir = seen.get(dirPath);638                if (!seenInDir) {639                    seenInDir = createMap<true>();640                    seen.set(dirPath, seenInDir);641                }642                resolutions.forEach((resolution, name) => {643                    if (seenInDir!.has(name)) {644                        return;645                    }646                    seenInDir!.set(name, true);647                    if (!resolution.isInvalidated && isInvalidatedResolution(resolution, getResolutionWithResolvedFileName)) {648                        // Mark the file as needing re-evaluation of module resolution instead of using it blindly.649                        resolution.isInvalidated = true;650                        (filesWithInvalidatedResolutions || (filesWithInvalidatedResolutions = createMap<true>())).set(containingFilePath, true);651                    }652                });653            });654        }655656        function hasReachedResolutionIterationLimit() {657            const maxSize = resolutionHost.maxNumberOfFilesToIterateForInvalidation || maxNumberOfFilesToIterateForInvalidation;658            return resolvedModuleNames.size > maxSize || resolvedTypeReferenceDirectives.size > maxSize;659        }660661        function invalidateResolutions(662            isInvalidatedResolution: (resolution: ResolutionWithFailedLookupLocations, getResolutionWithResolvedFileName: GetResolutionWithResolvedFileName) => boolean,663        ) {664            // If more than maxNumberOfFilesToIterateForInvalidation present,665            // just invalidated all files and recalculate the resolutions for files instead666            if (hasReachedResolutionIterationLimit()) {667                allFilesHaveInvalidatedResolution = true;668                return;669            }670            invalidateResolutionCache(resolvedModuleNames, isInvalidatedResolution, getResolvedModule);671            invalidateResolutionCache(resolvedTypeReferenceDirectives, isInvalidatedResolution, getResolvedTypeReferenceDirective);672        }673674        function invalidateResolutionOfFile(filePath: Path) {675            removeResolutionsOfFile(filePath);676            invalidateResolutions(677                // Resolution is invalidated if the resulting file name is same as the deleted file path678                (resolution, getResolutionWithResolvedFileName) => {679                    const result = getResolutionWithResolvedFileName(resolution);680                    return !!result && resolutionHost.toPath(result.resolvedFileName!) === filePath; // TODO: GH#18217681                }682            );683        }684685        function setFilesWithInvalidatedNonRelativeUnresolvedImports(filesMap: ReadonlyMap<ReadonlyArray<string>>) {686            Debug.assert(filesWithInvalidatedNonRelativeUnresolvedImports === filesMap || filesWithInvalidatedNonRelativeUnresolvedImports === undefined);687            filesWithInvalidatedNonRelativeUnresolvedImports = filesMap;688        }689690        function invalidateResolutionOfFailedLookupLocation(fileOrDirectoryPath: Path, isCreatingWatchedDirectory: boolean) {691            let isChangedFailedLookupLocation: (location: string) => boolean;692            if (isCreatingWatchedDirectory) {693                // Watching directory is created694                // Invalidate any resolution has failed lookup in this directory695                isChangedFailedLookupLocation = location => isInDirectoryPath(fileOrDirectoryPath, resolutionHost.toPath(location));696            }697            else {698                // If something to do with folder/file starting with "." in node_modules folder, skip it699                if (isPathInNodeModulesStartingWithDot(fileOrDirectoryPath)) return false;700701                // Some file or directory in the watching directory is created702                // Return early if it does not have any of the watching extension or not the custom failed lookup path703                const dirOfFileOrDirectory = getDirectoryPath(fileOrDirectoryPath);704                if (isNodeModulesAtTypesDirectory(fileOrDirectoryPath) || isNodeModulesDirectory(fileOrDirectoryPath) ||705                    isNodeModulesAtTypesDirectory(dirOfFileOrDirectory) || isNodeModulesDirectory(dirOfFileOrDirectory)) {706                    // Invalidate any resolution from this directory707                    isChangedFailedLookupLocation = location => {708                        const locationPath = resolutionHost.toPath(location);709                        return locationPath === fileOrDirectoryPath || startsWith(resolutionHost.toPath(location), fileOrDirectoryPath);710                    };711                }712                else {713                    if (!isPathWithDefaultFailedLookupExtension(fileOrDirectoryPath) && !customFailedLookupPaths.has(fileOrDirectoryPath)) {714                        return false;715                    }716                    // Ignore emits from the program717                    if (isEmittedFileOfProgram(resolutionHost.getCurrentProgram(), fileOrDirectoryPath)) {718                        return false;719                    }720                    // Resolution need to be invalidated if failed lookup location is same as the file or directory getting created721                    isChangedFailedLookupLocation = location => resolutionHost.toPath(location) === fileOrDirectoryPath;722                }723            }724            const hasChangedFailedLookupLocation = (resolution: ResolutionWithFailedLookupLocations) => some(resolution.failedLookupLocations, isChangedFailedLookupLocation);725            const invalidatedFilesCount = filesWithInvalidatedResolutions && filesWithInvalidatedResolutions.size;726            invalidateResolutions(727                // Resolution is invalidated if the resulting file name is same as the deleted file path728                hasChangedFailedLookupLocation729            );730            return allFilesHaveInvalidatedResolution || filesWithInvalidatedResolutions && filesWithInvalidatedResolutions.size !== invalidatedFilesCount;731        }732733        function closeTypeRootsWatch() {734            clearMap(typeRootsWatches, closeFileWatcher);735        }736737        function getDirectoryToWatchFailedLookupLocationFromTypeRoot(typeRoot: string, typeRootPath: Path): Path | undefined {738            if (allFilesHaveInvalidatedResolution) {739                return undefined;740            }741742            if (isInDirectoryPath(rootPath, typeRootPath)) {743                return rootPath;744            }745            const toWatch = getDirectoryToWatchFromFailedLookupLocationDirectory(typeRoot, typeRootPath);746            return toWatch && directoryWatchesOfFailedLookups.has(toWatch.dirPath) ? toWatch.dirPath : undefined;747        }748749        function createTypeRootsWatch(typeRootPath: Path, typeRoot: string): FileWatcher {750            // Create new watch and recursive info751            return resolutionHost.watchTypeRootsDirectory(typeRoot, fileOrDirectory => {752                const fileOrDirectoryPath = resolutionHost.toPath(fileOrDirectory);753                if (cachedDirectoryStructureHost) {754                    // Since the file existence changed, update the sourceFiles cache755                    cachedDirectoryStructureHost.addOrDeleteFileOrDirectory(fileOrDirectory, fileOrDirectoryPath);756                }757758                // For now just recompile759                // We could potentially store more data here about whether it was/would be really be used or not760                // and with that determine to trigger compilation but for now this is enough761                resolutionHost.onChangedAutomaticTypeDirectiveNames();762763                // Since directory watchers invoked are flaky, the failed lookup location events might not be triggered764                // So handle to failed lookup locations here as well to ensure we are invalidating resolutions765                const dirPath = getDirectoryToWatchFailedLookupLocationFromTypeRoot(typeRoot, typeRootPath);766                if (dirPath && invalidateResolutionOfFailedLookupLocation(fileOrDirectoryPath, dirPath === fileOrDirectoryPath)) {767                    resolutionHost.onInvalidatedResolution();768                }769            }, WatchDirectoryFlags.Recursive);770        }771772        /**773         * Watches the types that would get added as part of getAutomaticTypeDirectiveNames774         * To be called when compiler options change775         */776        function updateTypeRootsWatch() {777            const options = resolutionHost.getCompilationSettings();778            if (options.types) {779                // No need to do any watch since resolution cache is going to handle the failed lookups780                // for the types added by this781                closeTypeRootsWatch();782                return;783            }784785            // we need to assume the directories exist to ensure that we can get all the type root directories that get included786            // But filter directories that are at root level to say directory doesnt exist, so that we arent watching them787            const typeRoots = getEffectiveTypeRoots(options, { directoryExists: directoryExistsForTypeRootWatch, getCurrentDirectory });788            if (typeRoots) {789                mutateMap(790                    typeRootsWatches,791                    arrayToMap(typeRoots, tr => resolutionHost.toPath(tr)),792                    {793                        createNewValue: createTypeRootsWatch,794                        onDeleteValue: closeFileWatcher795                    }796                );797            }798            else {799                closeTypeRootsWatch();800            }801        }802803        /**804         * Use this function to return if directory exists to get type roots to watch805         * If we return directory exists then only the paths will be added to type roots806         * Hence return true for all directories except root directories which are filtered from watching807         */808        function directoryExistsForTypeRootWatch(nodeTypesDirectory: string) {809            const dir = getDirectoryPath(getDirectoryPath(nodeTypesDirectory));810            const dirPath = resolutionHost.toPath(dir);811            return dirPath === rootPath || canWatchDirectory(dirPath);812        }813    }
...

Resampling.py

Source:Resampling.py

1# -*- coding: utf-8 -*-2"""3The *Resampling* module provides methods to resample and reorient volumetric 4and point data. 5Resampling the data is usually necessary as the first step to match the 6resolution and orientation of the reference object. 7Main routines for resampling are: :func:`~ClearMap.Alignment.Resampling.resampleData` 8and :func:`~ClearMap.Alignment.Resampling.resamplePoints`.9Image Representation and Size10-----------------------------11The module assumes that images in arrays are arranged as 12    * [x,y] or 13    * [x,y,z] 14where x,y,z correspond to the x,y,z coordinates as displayed in e.g. ImageJ. 15For example an image of size (512,512) stored in an array ``img`` will have:16    >>> img.shape17    (512,512)18Points are assumed to be given as x,y,z coordinates19Parameters such as *resolution* or *dataSize* are assumed to be given in (x,y)20or (x,y,z) format, e.g.21    >>> dataSize = (512,512)22Orientation23-----------24The *orientation* parameter is a tuple of d numbers from 1 to d that specifies 25the permutation of the axes, a minus sign infront of a numbeer indicates26inversion of that axes. For exmaple 27    >>> orientation=(2,-1) 28indicates that x and y should be exchanged and the new y axes should be reversed.29Generally a re-orientation is composed of first a permutation of the axes and then30inverting the indicated axes.31A *permutation* is an orientation without signs and with numbers from 0 to d-1.32Examples:33    >>> import os34    >>> import ClearMap.IO as io35    >>> from ClearMap.Settings import ClearMapPath  36    >>> from ClearMap.Alignment.Resampling import resampleData37    >>> filename = os.path.join(ClearMapPath,'Test/Data/OME/16-17-27_0_8X-s3-20HF_UltraII_C00_xyz-Table Z\d{4}.ome.tif');38    >>> print io.dataSize(filename)39    (2160, 2560, 21)40    >>> data = resampleData(filename, sink = None, resolutionSource = (1,1,1), orientation = (1,2,3), resolutionSink = (10,10,2));41    >>> print data.shape42    (216, 256, 10)43"""44#:copyright: Copyright 2015 by Christoph Kirst, The Rockefeller University, New York City45#:license: GNU, see LICENSE.txt for details.46import sys, imp47import os48import math49import numpy50import multiprocessing  51import tempfile52import shutil53import cv254#import matplotlib.pyplot as plt55import ClearMap.IO.IO as io56import ClearMap.IO.FileList as fl57from ClearMap.Utils.ProcessWriter import ProcessWriter;58def fixOrientation(orientation):59    """Convert orientation to standard format number sequence60    61    Arguments:62        orientation (tuple or str): orientation specification63        64    Returns:65        tuple: orientation sequence66    67    See Also:68        `Orientation`_69    """70    71    if orientation is None:72        return None;73        74    #fix named representations75    if orientation == 'Left':76        orientation = (1,2,3);77    if orientation == 'Right':78        orientation = (-1,2,3);    79    80    return orientation;81def inverseOrientation(orientation):82    """Returns the inverse permuation of the permutation orientation taking axis inversions into account.83    84    Arguments:85        orientation (tuple or str): orientation specification86        87    Returns:88        tuple: orientation sequence89        90    See Also:91        `Orientation`_92    """93    94    if orientation is None:95        return None;96    97    n = len(orientation);98    iper = list(orientation);99    100    #permutation is defined as permuting the axes and then axis inversion101    for i in range(n):102        if orientation[i] < 0:103            iper[int(abs(orientation[i])-1)] = -(i + 1);104        else:105            iper[int(abs(orientation[i])-1)] = (i + 1);106    107    return tuple(iper)108def orientationToPermuation(orientation):109    """Extracts the permuation from an orientation.110    111    Arguments:112        orientation (tuple or str): orientation specification113        114    Returns:115        tuple: premutation sequence116        117    See Also:118        `Orientation`_119    """120    orientation = fixOrientation(orientation);121    if orientation is None:122        return (0,1,2);123    else:124        return tuple(int(abs(i))-1 for i in orientation);125def orientResolution(resolution, orientation):126    """Permutes a resolution tuple according to the given orientation.127    128    Arguments:129        resolution (tuple): resolution specification130        orientation (tuple or str): orientation specification131        132    Returns:133        tuple: oriented resolution sequence134        135    See Also:136        `Orientation`_137    """138    if resolution is None:139        return None;140    141    per = orientationToPermuation(orientation);142    #print orientation, per, resolution143    return tuple(resolution[i] for i in per);144    145def orientResolutionInverse(resolution, orientation):146    """Permutes a resolution tuple according to the inverse of a given orientation.147    148    Arguments:149        resolution (tuple): resolution specification150        orientation (tuple or str): orientation specification151        152    Returns:153        tuple: oriented resolution sequence154        155    See Also:156        `Orientation`_157    """158        159    if resolution is None:160        return None;161    162    per = orientationToPermuation(inverseOrientation(orientation));163    return tuple(resolution[i] for i in per);164 165def orientDataSize(dataSize, orientation):166    """Permutes a data size tuple according to the given orientation.167    168    Arguments:169        dataSize (tuple): resolution specification170        orientation (tuple or str): orientation specification171        172    Returns:173        tuple: oriented dataSize sequence174        175    See Also:176        `Orientation`_177    """178    179    return orientResolution(dataSize, orientation);180 181def orientDataSizeInverse(dataSize, orientation):182    """Permutes a dataSize tuple according to the inverse of a given orientation.183    184    Arguments:185        dataSize (tuple): dataSize specification186        orientation (tuple or str): orientation specification187        188    Returns:189        tuple: oriented dataSize sequence190        191    See Also:192        `Orientation`_193    """194    195    return orientResolutionInverse(dataSize, orientation); 196 197 198def resampleDataSize(dataSizeSource, dataSizeSink = None, resolutionSource = None, resolutionSink = None, orientation = None):199    """Calculate scaling factors and data sizes for resampling.200    201    Arguments:202        dataSizeSource (tuple): data size of the original image203        dataSizeSink (tuple or None): data size of the resmapled image204        resolutionSource (tuple or None): resolution of the source image205        resolutionSink (tuple or None): resolution of the sink image206        orientation (tuple or str): re-orientation specification207        208    Returns:209        tuple: data size of the source210        tuple: data size of the sink211        tuple: resolution of source212        tuple: resolution of sink213    214    See Also:215        `Orientation`_216    """217    orientation = fixOrientation(orientation);    218    219    #determine data sizes if not specified220    if dataSizeSink is None:221        if resolutionSource is None or resolutionSink is None:222            raise RuntimeError('resampleDataSize: data size and resolutions not defined!');223        224        #orient resolution of source to resolution of sink to get sink data size225        resolutionSourceO = orientResolution(resolutionSource, orientation);226        dataSizeSourceO = orientDataSize(dataSizeSource, orientation);227        228        #calculate scaling factor229        dataSizeSink = tuple([int(math.ceil(dataSizeSourceO[i] *  resolutionSourceO[i]/resolutionSink[i])) for i in range(len(dataSizeSource))]);        230        231    #print dataSizeSink, "ds sink"232    233    if dataSizeSource is None:234        if resolutionSource is None or resolutionSink is None:235            raise RuntimeError('resampleDataSize: data size and resolutions not defined!');236        237        #orient resolution of source to resolution of sink to get sink data size238        resolutionSourceO = orientResolution(resolutionSource, orientation);239        240        #calculate source data size241        dataSizeSource = tuple([int(math.ceil(dataSizeSink[i] *  resolutionSink[i]/resolutionSourceO[i])) for i in range(len(dataSizeSink))]);  242        dataSizeSource = orientDataSizeInverse(dataSizeSource);243        244    #print dataSizeSource, "ds source"245        246    #calculate effecive resolutions247    if resolutionSource is None:248        if resolutionSink is None:249            resolutionSource = (1,1,1);250        else:251            dataSizeSourceO = orientDataSize(dataSizeSource, orientation);252            resolutionSource = tuple(float(dataSizeSink[i]) / dataSizeSourceO[i] * resolutionSink[i] for i in range(len(dataSizeSource)));253            resolutionSource = orientResolutionInverse(resolutionSource, orientation);254    255    #print resolutionSource, "res source sink"256    257    dataSizeSourceO = orientDataSize(dataSizeSource, orientation);258    259    260    resolutionSourceO = orientResolution(resolutionSource, orientation);261    resolutionSink = tuple(float(dataSizeSourceO[i]) / float(dataSizeSink[i]) * resolutionSourceO[i] for i in range(len(dataSizeSource)));262    263    #print dataSizeSource, dataSizeSink, resolutionSource, resolutionSink 264    265    return dataSizeSource, dataSizeSink, resolutionSource, resolutionSink  266def fixInterpolation(interpolation):267    """Converts interpolation given as string to cv2 interpolation object268    269    Arguments:270        interpolation (str or object): interpolation string or cv2 object271    272    Returns:273        object: cv2 interpolation type274    """275    276    if interpolation == 'nn' or interpolation is None or interpolation == cv2.INTER_NEAREST:277        interpolation = cv2.INTER_NEAREST;278    else:279        interpolation = cv2.INTER_LINEAR;280        281    return interpolation;282        283def resampleXY(source, dataSizeSink, sink = None, interpolation = 'linear', out = sys.stdout, verbose = True):284    """Resample a 2d image slice285    286    This routine is used for resampling a large stack in parallel in xy or xz direction.287    288    Arguments:289        source (str or array): 2d image source290        dataSizeSink (tuple): size of the resmapled image291        sink (str or None): location for the resmapled image292        interpolation (str): interpolation method to use: 'linear' or None (nearest pixel)293        out (stdout): where to write progress information294        vebose (bool): write progress info if true295    296    Returns:297        array or str: resampled data or file name298    """   299    300    #out.write("Input: %s Output: " % (inputFile, soutputFile))301    data = io.readData(source);302    dataSize = data.shape;303    304    #print dataSize, dataSizeSink    305    306    if data.ndim != 2:307        raise RuntimeError('resampleXY: expects 2d image source, found %dd' % data.ndim)308    #print sagittalImageSize;309    310    #dataSizeSink = tuple([int(math.ceil(dataSize[i] *  resolutionSource[i]/resolutionSink[i])) for i in range(2)]);311    if verbose:312        out.write(("resampleData: Imagesize: %d, %d " % (dataSize[0], dataSize[1])) + ("Resampled Imagesize: %d, %d" % (dataSizeSink[0], dataSizeSink[1])))313        #out.write(("resampleData: Imagesize: %d, %d " % dataSize) + ("Resampled Imagesize: %d, %d" % (outputSize[1], outputSize[0])))314    315    # note: cv2.resize reverses x-Y axes316    interpolation = fixInterpolation(interpolation)317    sinkData = cv2.resize(data,  (dataSizeSink[1], dataSizeSink[0]), interpolation = interpolation);318    #sinkData = cv2.resize(data,  outputSize);319    #sinkData = scipy.misc.imresize(sagittalImage, outputImageSize, interp = 'bilinear'); #normalizes images -> not usefull for stacks !320    321    #out.write("resampleData: resized Image size: %d, %d " % sinkData.shape)322    323    return io.writeData(sink, sinkData);324def _resampleXYParallel(arg):325    """Resampling helper function to use for parallel resampling of image slices"""326    327    fileSource = arg[0];328    fileSink = arg[1];329    dataSizeSink = arg[2];330    interpolation = arg[3];331    ii = arg[4];332    nn = arg[5];333    verbose = arg[6];334    335    pw = ProcessWriter(ii);336    if verbose:337        pw.write("resampleData: resampling in XY: image %d / %d" % (ii, nn))338    339    data = numpy.squeeze(io.readData(fileSource, z = ii));340    resampleXY(data, sink = fileSink, dataSizeSink = dataSizeSink, interpolation = interpolation, out = pw, verbose = verbose);341def resampleData(source, sink = None,  orientation = None, dataSizeSink = None, resolutionSource = (4.0625, 4.0625, 3), resolutionSink = (25, 25, 25), 342                 processingDirectory = None, processes = 1, cleanup = True, verbose = True, interpolation = 'linear', **args):343    """Resample data of source in resolution and orientation344    345    Arguments:346        source (str or array): image to be resampled347        sink (str or None): destination of resampled image348        orientation (tuple): orientation specified by permuation and change in sign of (1,2,3)349        dataSizeSink (tuple or None): target size of the resampled image350        resolutionSource (tuple): resolution of the source image (in length per pixel)351        resolutionSink (tuple): resolution of the resampled image (in length per pixel)352        processingDirectory (str or None): directory in which to perform resmapling in parallel, None a temporary directry will be created353        processes (int): number of processes to use for parallel resampling354        cleanup (bool): remove temporary files355        verbose (bool): display progress information356        interpolation (str): method to use for interpolating to the resmapled image357    358    Returns:359        (array or str): data or file name of resampled image360    Notes: 361        * resolutions are assumed to be given for the axes of the intrinsic 362          orientation of the data and reference as when viewed by matplotlib or ImageJ363        * orientation: permuation of 1,2,3 with potential sign, indicating which 364          axes map onto the reference axes, a negative sign indicates reversal 365          of that particular axes366        * only a minimal set of information to detremine the resampling parameter 367          has to be given, e.g. dataSizeSource and dataSizeSink368    """369        370    orientation = fixOrientation(orientation);371    372    if isinstance(dataSizeSink, str):373        dataSizeSink = io.dataSize(dataSizeSink);374    #orient actual resolutions onto reference resolution    375    dataSizeSource = io.dataSize(source);376        377    dataSizeSource, dataSizeSink, resolutionSource, resolutionSink = resampleDataSize(dataSizeSource = dataSizeSource, dataSizeSink = dataSizeSink, 378                                                                                      resolutionSource = resolutionSource, resolutionSink = resolutionSink, orientation = orientation);379    380    dataSizeSinkI = orientDataSizeInverse(dataSizeSink, orientation);381    382    #print dataSizeSource, dataSizeSink, resolutionSource, resolutionSink, dataSizeSinkI383    384     385    #rescale in x y in parallel386    if processingDirectory == None:387        processingDirectory = tempfile.mkdtemp();     388        389    interpolation = fixInterpolation(interpolation);390     391    nZ = dataSizeSource[2];392    pool = multiprocessing.Pool(processes=processes);393    argdata = [];394    for i in range(nZ):395        argdata.append( (source, os.path.join(processingDirectory, 'resample_%04d.tif' % i), dataSizeSinkI, interpolation, i, nZ, verbose) );  396        #print argdata[i]397    pool.map(_resampleXYParallel, argdata);398    399    #rescale in z400    fn = os.path.join(processingDirectory, 'resample_%04d.tif' % 0);401    data = io.readData(fn);402    zImage = numpy.zeros((dataSizeSinkI[0], dataSizeSinkI[1], nZ), dtype = data.dtype);    403    for i in range(nZ):404        if verbose and i % 10 == 0:405            print("resampleData; reading %d/%d" % (i, nZ));406        fn = os.path.join(processingDirectory, 'resample_%04d.tif' % i);407        zImage[:,:, i] = io.readData(fn);408    409    resampledData = numpy.zeros(dataSizeSinkI, dtype = zImage.dtype);410    for i in range(dataSizeSinkI[0]):411        if verbose and i % 25 == 0:412            print("resampleData: processing %d/%d" % (i, dataSizeSinkI[0]))413        #resampledImage[:, iImage ,:] =  scipy.misc.imresize(zImage[:,iImage,:], [resizedZAxisSize, sagittalImageSize[1]] , interp = 'bilinear'); 414        #cv2.resize takes reverse order of sizes !415        resampledData[i ,:, :] =  cv2.resize(zImage[i,:,:], (dataSizeSinkI[2], dataSizeSinkI[1]), interpolation = interpolation);416        #resampledData[i ,:, :] =  cv2.resize(zImage[i,:, :], (dataSize[1], resizedZSize));417    418    #account for using (z,y,x) array representation -> (y,x,z)419    #resampledData = resampledData.transpose([1,2,0]);420    #resampledData = resampledData.transpose([2,1,0]);421    422    if cleanup:423        shutil.rmtree(processingDirectory);424    if not orientation is None:425        426        #reorient427        per = orientationToPermuation(orientation);428        resampledData = resampledData.transpose(per);429    430        #reverse orientation after permuting e.g. (-2,1) brings axis 2 to first axis and we can reorder there431        if orientation[0] < 0:432            resampledData = resampledData[::-1, :, :];433        if orientation[1] < 0:434            resampledData = resampledData[:, ::-1, :]; 435        if orientation[2] < 0:436            resampledData = resampledData[:, :, ::-1];437        438        #bring back from y,x,z to z,y,x439        #resampledImage = resampledImage.transpose([2,0,1]);440    if verbose:441        print("resampleData: resampled data size: " + str(resampledData.shape))442    443    if sink == []:444        if io.isFileExpression(source):445            sink = os.path.split(source);446            sink = os.path.join(sink[0], 'resample_\d{4}.tif');447        elif isinstance(source, str):448            sink = source + '_resample.tif';449        else:450            raise RuntimeError('resampleData: automatic sink naming not supported for non string source!');451    452    return io.writeData(sink, resampledData);453    454    455    456def resampleDataInverse(sink, source = None, dataSizeSource = None, orientation = None, resolutionSource = (4.0625, 4.0625, 3), resolutionSink = (25, 25, 25), 457                        processingDirectory = None, processes = 1, cleanup = True, verbose = True, interpolation = 'linear', **args):458    """Resample data inversely to :func:`resampleData` routine459    460    Arguments:461        sink (str or None): image to be inversly resampled (=sink in :func:`resampleData`)462        source (str or array): destination for inversly resmapled image (=source in :func:`resampleData`)463        dataSizeSource (tuple or None): target size of the resampled image464        orientation (tuple): orientation specified by permuation and change in sign of (1,2,3)465        resolutionSource (tuple): resolution of the source image (in length per pixel)466        resolutionSink (tuple): resolution of the resampled image (in length per pixel)467        processingDirectory (str or None): directory in which to perform resmapling in parallel, None a temporary directry will be created468        processes (int): number of processes to use for parallel resampling469        cleanup (bool): remove temporary files470        verbose (bool): display progress information471        interpolation (str): method to use for interpolating to the resmapled image472    473    Returns:474        (array or str): data or file name of resampled image475    Notes: 476        * resolutions are assumed to be given for the axes of the intrinsic 477          orientation of the data and reference as when viewed by matplotlib or ImageJ478        * orientation: permuation of 1,2,3 with potential sign, indicating which 479          axes map onto the reference axes, a negative sign indicates reversal 480          of that particular axes481        * only a minimal set of information to detremine the resampling parameter 482          has to be given, e.g. dataSizeSource and dataSizeSink483    """    484    485    486    #orientation487    orientation = fixOrientation(orientation);488    489    #assume we can read data fully into memory490    resampledData = io.readData(sink);491    dataSizeSink = resampledData.shape;492    493    if isinstance(dataSizeSource, str):494        dataSizeSource = io.dataSize(dataSizeSource);495    dataSizeSource, dataSizeSink, resolutionSource, resolutionSink = resampleDataSize(dataSizeSource = dataSizeSource, dataSizeSink = dataSizeSink, 496                                                                                      resolutionSource = resolutionSource, resolutionSink = resolutionSink, orientation = orientation);497    #print (dataSizeSource, dataSizeSink, resolutionSource, resolutionSink )498    499    dataSizeSinkI = orientDataSizeInverse(dataSizeSink, orientation);500    501    502    #flip axes back and permute inversely503    if not orientation is None:504        if orientation[0] < 0:505            resampledData = resampledData[::-1, :, :];506        if orientation[1] < 0:507            resampledData = resampledData[:, ::-1, :]; 508        if orientation[2] < 0:509            resampledData = resampledData[:, :, ::-1];510        511        #reorient512        peri = inverseOrientation(orientation);513        peri = orientationToPermuation(peri);514        resampledData = resampledData.transpose(peri);515    516    # upscale in z517    interpolation = fixInterpolation(interpolation);518    519    resampledDataXY = numpy.zeros((dataSizeSinkI[0], dataSizeSinkI[1], dataSizeSource[2]), dtype = resampledData.dtype);    520    521    for i in range(dataSizeSinkI[0]):522        if verbose and i % 25 == 0:523            print("resampleDataInverse: processing %d/%d" % (i, dataSizeSinkI[0]))524        #cv2.resize takes reverse order of sizes !525        resampledDataXY[i ,:, :] =  cv2.resize(resampledData[i,:,:], (dataSizeSource[2], dataSizeSinkI[1]), interpolation = interpolation);526    # upscale x, y in parallel527    528    if io.isFileExpression(source):529        files = source;530    else:531        if processingDirectory == None:532            processingDirectory = tempfile.mkdtemp();   533        files = os.path.join(sink[0], 'resample_\d{4}.tif');534    535    io.writeData(files, resampledDataXY);536    537    nZ = dataSizeSource[2];538    pool = multiprocessing.Pool(processes=processes);539    argdata = [];540    for i in range(nZ):541        argdata.append( (source, fl.fileExpressionToFileName(files, i), dataSizeSource, interpolation, i, nZ) );  542    pool.map(_resampleXYParallel, argdata);543    544    if io.isFileExpression(source):545        return source;546    else:547        data = io.convertData(files, source);548        549        if cleanup:550            shutil.rmtree(processingDirectory);551        552        return data;553    554def resamplePoints(pointSource, pointSink = None, dataSizeSource = None, dataSizeSink = None, orientation = None, resolutionSource = (4.0625, 4.0625, 3), resolutionSink = (25, 25, 25), **args):555    """Resample Points to map from original data to the coordinates of the resampled image556    557    The resampling of points here corresponds to he resampling of an image in :func:`resampleData`558        559    Arguments:560        pointSource (str or array): image to be resampled561        pointSink (str or None): destination of resampled image562        orientation (tuple): orientation specified by permuation and change in sign of (1,2,3)563        dataSizeSource (str, tuple or None): size of the data source564        dataSizeSink (str, tuple or None): target size of the resampled image565        resolutionSource (tuple): resolution of the source image (in length per pixel)566        resolutionSink (tuple): resolution of the resampled image (in length per pixel)567        568    Returns:569        (array or str): data or file name of resampled points570    Notes: 571        * resolutions are assumed to be given for the axes of the intrinsic 572          orientation of the data and reference as when viewed by matplotlib or ImageJ573        * orientation: permuation of 1,2,3 with potential sign, indicating which 574          axes map onto the reference axes, a negative sign indicates reversal 575          of that particular axes576        * only a minimal set of information to detremine the resampling parameter 577          has to be given, e.g. dataSizeSource and dataSizeSink578    """579    580    #fix (y,x,z) image array representation581    #resolutionSource, resolutionSink = self.fixResolutions(resolutionSource, resolutionSink);582    583    orientation = fixOrientation(orientation);584    #datasize of data source585    if isinstance(dataSizeSource, str):586        dataSizeSource = io.dataSize(dataSizeSource);587    588    dataSizeSource, dataSizeSink, resolutionSource, resolutionSink = resampleDataSize(dataSizeSource = dataSizeSource, dataSizeSink = dataSizeSink, 589                                                                                      resolutionSource = resolutionSource, resolutionSink = resolutionSink, orientation = orientation);590    points = io.readPoints(pointSource);591    dataSizeSinkI = orientDataSizeInverse(dataSizeSink, orientation);592    #resolutionSinkI = orientResolutionInverse(resolutionSink, orientation);593        594    #scaling factors595    scale = [float(dataSizeSource[i]) / float(dataSizeSinkI[i]) for i in range(3)];596    #print scale597    598    repoints = points.copy();599    for i in range(3):    600        repoints[:,i] = repoints[:,i] / scale[i];601               602    #permute for non trivial orientation603    if not orientation is None:604        per = orientationToPermuation(orientation);605        repoints = repoints[:,per];606        607        for i in range(3):608            if orientation[i] < 0:609                repoints[:,i] = dataSizeSink[i] - repoints[:,i];610      611    return io.writePoints(pointSink, repoints);612     613def resamplePointsInverse(pointSource, pointSink = None, dataSizeSource = None, dataSizeSink = None, orientation = None, resolutionSource = (4.0625, 4.0625, 3), resolutionSink = (25, 25, 25), **args):614    """Resample points from the coordinates of the resampled image to the original data615    The resampling of points here corresponds to he resampling of an image in :func:`resampleDataInverse`616        617    Arguments:618        pointSource (str or array): image to be resampled619        pointSink (str or None): destination of resampled image620        orientation (tuple): orientation specified by permuation and change in sign of (1,2,3)621        dataSizeSource (str, tuple or None): size of the data source622        dataSizeSink (str, tuple or None): target size of the resampled image623        resolutionSource (tuple): resolution of the source image (in length per pixel)624        resolutionSink (tuple): resolution of the resampled image (in length per pixel)625        626    Returns:627        (array or str): data or file name of inversely resampled points628    Notes: 629        * resolutions are assumed to be given for the axes of the intrinsic 630          orientation of the data and reference as when viewed by matplotlib or ImageJ631        * orientation: permuation of 1,2,3 with potential sign, indicating which 632          axes map onto the reference axes, a negative sign indicates reversal 633          of that particular axes634        * only a minimal set of information to detremine the resampling parameter 635          has to be given, e.g. dataSizeSource and dataSizeSink636    """637       638    orientation = fixOrientation(orientation);639    640    #datasize of data source641    if isinstance(dataSizeSource, str):642        dataSizeSource = io.dataSize(dataSizeSource);643    644    dataSizeSource, dataSizeSink, resolutionSource, resolutionSink = resampleDataSize(dataSizeSource = dataSizeSource, dataSizeSink = dataSizeSink, 645                                                                                      resolutionSource = resolutionSource, resolutionSink = resolutionSink, orientation = orientation);646            647    points = io.readPoints(pointSource);648    649    dataSizeSinkI = orientDataSizeInverse(dataSizeSink, orientation);650    #resolutionSinkI = orientResolutionInverse(resolutionSink, orientation);651        652    #scaling factors653    scale = [float(dataSizeSource[i]) / float(dataSizeSinkI[i]) for i in range(3)];654    #print scale655    rpoints = points.copy();    656    657    #invert axis inversion and permutations    658    if not orientation is None:659        #invert permuation660        iorientation = inverseOrientation(orientation);661        per = orientationToPermuation(iorientation);662        rpoints = rpoints[:,per];663        664        for i in range(3):665            if iorientation[i] < 0:666                rpoints[:,i] = dataSizeSink[i] - rpoints[:,i];667    668    #scale points669    for i in range(3):   670        rpoints[:,i] = rpoints[:,i] * scale[i];    671    672    return io.writePoints(pointSink, rpoints);673def sagittalToCoronalData(source, sink = None):674    """Change from saggital to coronal orientation675     676    Arguments:677        source (str or array): source data to be reoriented678        sink (str or None): destination for reoriented image679    680    Returns:681        str or array: reoriented data682    """683      684    source = io.readData(source);685    d = source.ndim;686    if d < 3:687        raise RuntimeError('sagittalToCoronalData: 3d image required!');688    689    tp = range(d);690    tp[0:3] = [2,0,1];691    source = source.transpose(tp);692    source = source[::-1];693    #source = source[::-1,:,:];694    return io.writeData(sink, source);695def _test():696    """Tests for the Resampling Module"""697    import ClearMap.Alignment.Resampling as self698    imp.reload(self)699    from ClearMap.Settings import ClearMapPath as basedir 700    import iDISCO.IO.IO as io701    import os, numpy702    fn = os.path.join(basedir, 'Test/Data/OME/16-17-27_0_8X-s3-20HF_UltraII_C00_xyz-Table Z\d{4}.ome.tif');703    outfn = os.path.join(basedir, "Test/Data/Resampling/test.mhd")704    705    print("Making resampled stack " + outfn)706    print("source datasize %s" % str(io.dataSize(fn)));707    data = self.resampleData(fn, sink = None, resolutionSource = (1,1,1), orientation = (1,2,3), resolutionSink = (10,10,2));708    print(data.shape)709    io.writeData(outfn, data)   710    data = self.resampleData(fn, sink = None, dataSizeSink = (50,70,10), orientation = (1,2,3));711    print(data.shape)712    io.writeData(outfn, data)   713    dataSizeSource, dataSizeSink, resolutionSource, resolutionSink = self.resampleDataSize(dataSizeSource = (100,200, 303), dataSizeSink = None, 714                                                                                      resolutionSource = (1,1,1), resolutionSink = (5,5,5), orientation = (1,2,3));715    print(dataSizeSource, dataSizeSink, resolutionSource, resolutionSink)716    717    points = numpy.array([[0,0,0], [1,1,1], io.dataSize(fn)]);718    points = points.astype('float')719    pr = self.resamplePoints(points, dataSizeSource = fn, dataSizeSink = (50,70,10), orientation = (1,2,3))720    print(pr)721    pri = self.resamplePointsInverse(pr, dataSizeSource = fn, dataSizeSink = (50,70,10), orientation = (-1,2,3))722    print(pri)723    result = self.resampleDataInverse(outfn, os.path.join(basedir, 'Test/Data/OME/resample_\d{4}.ome.tif'), dataSizeSource = fn);724    print(result)725if __name__ == "__main__":...

CreateHaloFiles.py

Source:CreateHaloFiles.py

1#!/usr/bin/env conda-env-py32import tangos3import os4import numpy as np5import h5py6h = 0.67769427832679697from tangos_halo_module.path import get_file_path, get_halo_snap_num8from tangos_halo_module.halos import get_survivors, get_zombies, get_host, get_survivor_IDs, get_zombie_IDs, blockPrint, enablePrint9from tangos_halo_module.halo_properties import track_halo_property, get_timesteps10def track_halo_property(simulation, key, tangos_halo=0, halo_path=0, halo_id=0, snap_num=0, resolution=1000):11    '''12    input params: 13        simulation: h148, h229, h242, h329: string14        key: pre-existing tangos halo property (eg. 'Mvir' or 'VXc'): string15        tangos_halo: a valid Tangos halo object16        halo_path: 0 or a complete tangos halo address: path or string object17        halo_id: halo id: string or numeric18        snap_num: 4 digit simulation snapshot number: string19    output params:20        tracked parameter: earliest --> latest snapshot: numpy array21    '''22    23    if tangos_halo:24        halo = tangos_halo25    elif halo_id and snap_num and simulation:26        #string tangos halo query from components27        if simulation == 'h148':28            halo = tangos.get_halo("snapshots/"+ str(simulation) +29                               ".cosmo50PLK.3072g3HbwK1BH.00"+ str(snap_num) +"/"+ str(halo_id))30        else:31            halo = tangos.get_halo("snapshots/"+ str(simulation) +32                               ".cosmo50PLK.3072gst5HbwK1BH.00"+ str(snap_num) +"/"+ str(halo_id))33    elif halo_path:34        halo = tangos.get_halo(str(halo))35    else:36        raise ValueError("Halo %r not found. You got this tho." % (halo))37        #edit .db path to match local address38    if resolution==1000:39        tangos.init_db("simulation_database/"+ str(simulation) +".db")40    elif resolution==100:41        tangos.init_db("/data/Sims/Databases/100Nptcls/"+ str(simulation) +".db")42    else:43        raise ValueError("Resolution not implemented yet.")44        45    # round all times in Gyr to 2 decimal places due to 46    # inconsistent float truncation between response to livecalculation query and direct database query47    halo_gyr = round(halo.timestep.time_gyr, 2)48    prog = np.flip(halo.calculate_for_progenitors(str(key))[0])49    desc = halo.calculate_for_descendants(str(key))[0][1:]50    track = np.array(np.concatenate((prog, desc), axis=None))51    len_track = len(track)52    timesteps = get_timesteps(simulation)[0] 53    timesteps = [round(t, 2) for t in timesteps] #in gyr since start of sim54    num_timesteps = len(timesteps)55    #len(tangos.get_simulation("snapshots").timesteps) #62 for h229, h242, h32956    57    if len_track == num_timesteps:58        return np.asarray(track)59    elif len_track > num_timesteps/2: #pad track with initial 0s60        return np.array(np.concatenate((np.zeros(num_timesteps - len_track), track), axis=None), dtype=float)61    else:62        # i = np.where(timesteps==halo_gyr)[0][0]63        i = timesteps.index(halo_gyr)64        new_track = np.zeros(num_timesteps)65        new_track[i+1:i+1+len(desc)] = desc66        new_track[i+1-len(prog):i+1] = prog67        return new_track 68    69def write_halo_file(halo, simulation, status, mode='add', resolution=100):70    import h5py71    h = 0.677694278326796972    73    if mode == 'write': # Write74        m = 'w'75    else: # Add76        m = 'a'77    path = get_file_path(tangos_halo=halo, simulation=simulation, status=status, halo_id=0, snap_num=0, resolution=resolution)78    with h5py.File(path, m) as f:79        d1 = f.create_dataset('time_|_Gyr', data = get_timesteps(simulation=simulation, resolution=resolution)[0])80        d2 = f.create_dataset('time_|_redshift', data = get_timesteps(simulation=simulation, resolution=resolution)[1])81        d3 = f.create_dataset('time_|_a', data = get_timesteps(simulation=simulation, resolution=resolution)[2])82#         d4 = f.create_dataset('SFR_100Myr_|_Msol/yr', data = track_halo_property(tangos_halo=halo, simulation=simulation, key='SFR_100Myr', resolution=resolution))83        d5 = f.create_dataset('Xc_|_kpc', data = track_halo_property(tangos_halo=halo, simulation=simulation, key='Xc', resolution=resolution)*a/h)84        d6 = f.create_dataset('Yc_|_kpc', data = track_halo_property(tangos_halo=halo, simulation=simulation, key='Yc', resolution=resolution)*a/h)85        d7 = f.create_dataset('Zc_|_kpc', data = track_halo_property(tangos_halo=halo, simulation=simulation, key='Zc', resolution=resolution)*a/h)86        d8 = f.create_dataset('Rvir_|_kpc', data = track_halo_property(tangos_halo=halo, simulation=simulation, key='Rvir', resolution=resolution)*a/h)87        d9 = f.create_dataset('Mvir_|_Msol', data = track_halo_property(tangos_halo=halo, simulation=simulation, key='Mvir', resolution=resolution)*h)88        d10 = f.create_dataset('M_star_|_Msol', data = track_halo_property(tangos_halo=halo, simulation=simulation, key='M_star', resolution=resolution)*h)89        d11 = f.create_dataset('VXc_|_km/s', data = track_halo_property(tangos_halo=halo, simulation=simulation, key='VXc', resolution=resolution))90        d12 = f.create_dataset('VYc_|_km/s', data = track_halo_property(tangos_halo=halo, simulation=simulation, key='VYc', resolution=resolution))91        d13 = f.create_dataset('VZc_|_km/s', data = track_halo_property(tangos_halo=halo, simulation=simulation, key='VZc', resolution=resolution))92        d14 = f.create_dataset('n_gas', data = track_halo_property(tangos_halo=halo, simulation=simulation, key='n_gas', resolution=resolution))93        d15 = f.create_dataset('n_star', data = track_halo_property(tangos_halo=halo, simulation=simulation, key='n_star', resolution=resolution))94        d16 = f.create_dataset('n_dm', data = track_halo_property(tangos_halo=halo, simulation=simulation, key='n_dm', resolution=resolution))95        d17 = f.create_dataset('M_gas_|_Msol', data = track_halo_property(tangos_halo=halo, simulation=simulation, key='M_gas', resolution=resolution)*h)96    return97# Write data98for sim in ['h148', 'h229', 'h242', 'h329']:99    100    # Host101    print('Started Host: ' + str(sim))102    halo = get_host(simulation=sim, resolution=100)103    a = get_timesteps(simulation=sim, resolution=100)[2]104    write_halo_file(halo=halo, simulation=sim, status='Host', resolution=100, mode='write')105    print('Finished with Host: ' + str(sim))106    107    print('Started Survivors: ' + str(sim))108    survivors = get_survivors(simulation=sim, resolution=100)109    for sat in survivors:110        write_halo_file(halo=sat, simulation=sim, status='Survivor', resolution=100, mode='write')111    print('Finished with Survivors: ' + str(sim))112    113    print('Started Zombies: ' + str(sim))114    zombies = get_zombies(simulation=sim, resolution=100)115    for sat in zombies:116        write_halo_file(halo=sat, simulation=sim, status='Zombie', resolution=100, mode='write')...

pg_rec_losses.py

Source:pg_rec_losses.py

...8        self._resolution = max_resolution9        self._stage = ProgGrowStageType.stab10        self._progress = 011    @abstractmethod12    def set_stage_resolution(self, stage, resolution):13        pass14    @abstractmethod15    def set_progress(self, progress):16        pass17    @abstractmethod18    def forward(self, x, y):19        pass20    @abstractmethod21    def set_reduction(self, reduction):22        pass23class PGPerceptualLoss(AbstractPGLoss):24    def __init__(self, max_resolution, weights_per_resolution,25                 reduction='mean',26                 use_smooth_pg=False,27                 use_feature_normalization=False,28                 use_L1_norm=False,29                 use_relative_error=False):30        super(PGPerceptualLoss, self).__init__(max_resolution)31        self._max_resolution = max_resolution32        self._weights_per_resolution = weights_per_resolution33        self._use_smooth_pg = use_smooth_pg34        self._loss = PerceptualLoss(reduction=reduction,35                                    use_feature_normalization=use_feature_normalization,36                                    use_L1_norm=use_L1_norm,37                                    use_relative_error=use_relative_error)38        self._resolution = self._max_resolution39        self._stage = ProgGrowStageType.stab40        self._progress = 041    def set_stage_resolution(self, stage, resolution):42        self._stage = stage43        self._resolution = resolution44        self._progress = 045    def set_progress(self, progress):46        self._progress = progress47    def set_reduction(self, reduction):48        self._loss.reduction = reduction49    def forward(self, x, y):50        self._loss.set_new_weights(**self._weights_per_resolution[self._resolution])51        loss = self._loss(x, y)52        if self._use_smooth_pg:53            if self._stage == ProgGrowStageType.trns and self._progress < 1:54                prev_res = int(self._resolution / 2)55                self._loss.set_new_weights(**self._weights_per_resolution[prev_res])56                x = torch.nn.functional.upsample(x, scale_factor=0.5, mode='bilinear')57                y = torch.nn.functional.upsample(y, scale_factor=0.5, mode='bilinear')58                prev_loss = self._loss(x, y)59                loss = (1 - self._progress) * prev_loss + self._progress * loss60        return loss61class PGRelativePerceptualL1Loss(PGPerceptualLoss):62    def __init__(self, max_resolution, weights_per_resolution, reduction='mean', use_smooth_pg=False):63        super().__init__(64            max_resolution, weights_per_resolution,65            reduction=reduction,66            use_smooth_pg=use_smooth_pg,67            use_feature_normalization=True,68            use_L1_norm=True,69            use_relative_error=True)70class PGL2Loss(AbstractPGLoss):71    def __init__(self, max_resolution, reduction='mean'):72        super().__init__(max_resolution)73        self._loss = L2Loss(reduction=reduction)74    def set_stage_resolution(self, stage, resolution):75        pass76    def set_progress(self, progress):77        pass78    def set_reduction(self, reduction):79        self._loss.set_reduction(reduction)80    def forward(self, x, y):81        return self._loss(x, y)82class PGL1Loss(AbstractPGLoss):83    def __init__(self, max_resolution, reduction='mean'):84        super().__init__(max_resolution)85        self._loss = L1Loss(reduction=reduction)86    def set_stage_resolution(self, stage, resolution):87        pass88    def set_progress(self, progress):89        pass90    def set_reduction(self, reduction):91        self._loss.set_reduction(reduction)92    def forward(self, x, y):93        return self._loss(x, y)94class PGComposeLoss(AbstractPGLoss):95    def __init__(self, max_resolution, loss_1, loss_2):96        super().__init__(max_resolution)97        self.loss_1 = PG_RECONSTRUCTION_LOSSES[ReconstructionLossType[loss_1['loss_type']]](98            max_resolution=max_resolution, **loss_1['loss_kwargs'])99        self.loss_1_weight = loss_1['loss_weight']100        self.loss_2 = PG_RECONSTRUCTION_LOSSES[ReconstructionLossType[loss_2['loss_type']]](101            max_resolution=max_resolution, **loss_2['loss_kwargs'])102        self.loss_2_weight = loss_2['loss_weight']103    def set_stage_resolution(self, stage, resolution):104        self.loss_1.set_stage_resolution(stage, resolution)105        self.loss_2.set_stage_resolution(stage, resolution)106    def set_progress(self, progress):107        self.loss_1.set_progress(progress)108        self.loss_2.set_progress(progress)109    def forward(self, x, y):110        return self.loss_1_weight * self.loss_1(x, y) + self.loss_2_weight * self.loss_2(x, y)111    def set_reduction(self, reduction):112        self.loss_1.set_reduction(reduction)113        self.loss_2.set_reduction(reduction)114PG_RECONSTRUCTION_LOSSES = {115    ReconstructionLossType.perceptual: PGPerceptualLoss,116    ReconstructionLossType.relative_perceptual_L1: PGRelativePerceptualL1Loss,117    ReconstructionLossType.l1: PGL1Loss,118    ReconstructionLossType.l2: PGL2Loss,119    ReconstructionLossType.compose: PGComposeLoss...

Automation Testing Tutorials

Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.