Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 15 additions & 5 deletions packages/js-sdk/src/template/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ import {
calculateFilesHash,
getCallerDirectory,
getCallerFrame,
normalizeCopySourcePath,
padOctal,
readDockerignore,
readGCPServiceAccountJSON,
Expand Down Expand Up @@ -357,8 +358,10 @@ export class TemplateBase
const srcs = Array.isArray(src) ? src : [src]

for (const src of srcs) {
const { normalizedSrc, contextPathForInstruction } =
normalizeCopySourcePath(src.toString(), this.fileContextPath)
const args = [
src.toString(),
normalizedSrc,
dest.toString(),
options?.user ?? '',
options?.mode ? padOctal(options.mode) : '',
Expand All @@ -370,6 +373,7 @@ export class TemplateBase
force: options?.forceUpload || this.forceNextLayer,
forceUpload: options?.forceUpload,
resolveSymlinks: options?.resolveSymlinks,
contextPath: contextPathForInstruction,
})
}

Expand Down Expand Up @@ -901,6 +905,9 @@ export class TemplateBase
throw new Error('Source path and files hash are required')
}

const contextPathForInstruction =
instruction.contextPath ?? this.fileContextPath.toString()

const forceUpload = instruction.forceUpload
let stackTrace = undefined
if (index + 1 >= 0 && index + 1 < this.stackTraces.length) {
Expand All @@ -923,11 +930,11 @@ export class TemplateBase
await uploadFile(
{
fileName: src,
fileContextPath: this.fileContextPath.toString(),
fileContextPath: contextPathForInstruction,
url,
ignorePatterns: [
...this.fileIgnorePatterns,
...readDockerignore(this.fileContextPath.toString()),
...readDockerignore(contextPathForInstruction),
],
resolveSymlinks: instruction.resolveSymlinks ?? RESOLVE_SYMLINKS,
},
Expand Down Expand Up @@ -990,6 +997,9 @@ export class TemplateBase
throw new Error('Source path and destination path are required')
}

const contextPathForInstruction =
instruction.contextPath ?? this.fileContextPath.toString()

let stackTrace = undefined
if (index + 1 >= 0 && index + 1 < this.stackTraces.length) {
stackTrace = this.stackTraces[index + 1]
Expand All @@ -1000,12 +1010,12 @@ export class TemplateBase
filesHash: await calculateFilesHash(
src,
dest,
this.fileContextPath.toString(),
contextPathForInstruction,
[
...this.fileIgnorePatterns,
...(runtime === 'browser'
? []
: readDockerignore(this.fileContextPath.toString())),
: readDockerignore(contextPathForInstruction)),
Comment thread
mishushakov marked this conversation as resolved.
],
instruction.resolveSymlinks ?? RESOLVE_SYMLINKS,
stackTrace
Expand Down
1 change: 1 addition & 0 deletions packages/js-sdk/src/template/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,7 @@ export type Instruction = {
forceUpload?: true
filesHash?: string
resolveSymlinks?: boolean
contextPath?: string
}

/**
Expand Down
74 changes: 66 additions & 8 deletions packages/js-sdk/src/template/utils.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import crypto from 'node:crypto'
import fs from 'node:fs'
import fs, { PathLike } from 'node:fs'
import path from 'node:path'
import { dynamicImport, dynamicRequire } from '../utils'
import { BASE_STEP_NAME, FINALIZE_STEP_NAME } from './consts'
Expand Down Expand Up @@ -33,6 +33,57 @@ function normalizePath(path: string): string {
return path.replace(/\\/g, '/')
}

/**
* Normalize a COPY source path into a context-relative pattern and its context.
*
* - Absolute sources: context is the source's directory; normalized path is the
* relative path from that directory ('.' when the source is the directory itself).
* - Relative sources: context defaults to `fileContextPath`; if the path escapes
* that context (e.g., '../../../foo'), use the escaped path's directory instead.
* - Always returns POSIX separators for glob/tar friendliness.
*
* @param src The source path to normalize
* @param fileContextPath The context path to use
* @returns An object containing the normalized source path and the context path
*/
export function normalizeCopySourcePath(
src: string,
fileContextPath: PathLike
): {
normalizedSrc: string
contextPathForInstruction: string
} {
const defaultContext = path.resolve(fileContextPath.toString())
const absoluteSrc = path.isAbsolute(src)
? src
: path.resolve(defaultContext, src)

// Absolute sources: keep full path structure in the archive by anchoring at '/'
if (path.isAbsolute(src)) {
const normalizedSrc = normalizePath(path.relative('/', absoluteSrc)) || '.'
return {
normalizedSrc,
contextPathForInstruction: '/',
}
Comment thread
mishushakov marked this conversation as resolved.
}

// Relative sources: prefer default context, but if they escape, anchor at '/'
const relativeToDefault = path.relative(defaultContext, absoluteSrc)
const escapesDefault =
relativeToDefault === '..' ||
relativeToDefault.startsWith(`..${path.sep}`) ||
relativeToDefault.startsWith('../')

const contextPathForInstruction = escapesDefault ? '/' : defaultContext
const normalizedSrc =
normalizePath(path.relative(contextPathForInstruction, absoluteSrc)) || '.'

return {
normalizedSrc,
contextPathForInstruction,
}
}

/**
* Get all files for a given path and ignore patterns.
*
Expand All @@ -49,12 +100,13 @@ export async function getAllFilesInPath(
) {
const { glob } = await dynamicImport<typeof import('glob')>('glob')
const files = new Map<string, Path>()
const isAbsoluteSrc = path.isAbsolute(src)

const globFiles = await glob(src, {
ignore: ignorePatterns,
withFileTypes: true,
// this is required so that the ignore pattern is relative to the file path
cwd: contextPath,
cwd: isAbsoluteSrc ? undefined : contextPath,
})

for (const file of globFiles) {
Expand All @@ -64,15 +116,17 @@ export async function getAllFilesInPath(
files.set(file.fullpath(), file)
}
const dirPattern = normalizePath(
// When the matched directory is '.', `file.relative()` can be an empty string.
// In that case, we want to match all files under the current directory instead of
// creating an absolute glob like '/**/*' which would traverse the entire filesystem.
path.join(file.relative() || '.', '**/*')
isAbsoluteSrc
? path.join(file.fullpath(), '**/*')
: // When the matched directory is '.', `file.relative()` can be an empty string.
// In that case, we want to match all files under the current directory instead of
// creating an absolute glob like '/**/*' which would traverse the entire filesystem.
path.join(file.relative() || '.', '**/*')
)
const dirFiles = await glob(dirPattern, {
ignore: ignorePatterns,
withFileTypes: true,
cwd: contextPath,
cwd: isAbsoluteSrc ? undefined : contextPath,
})
dirFiles.forEach((f) => files.set(f.fullpath(), f))
} else {
Expand Down Expand Up @@ -276,7 +330,11 @@ export async function tarFileStream(
true
)

const filePaths = allFiles.map((file) => file.relativePosix())
const filePaths = allFiles.map((file) => {
const rel = path.relative(fileContextPath, file.fullpath())
const normalized = normalizePath(rel || '.')
return normalized
})

return create(
{
Expand Down
17 changes: 17 additions & 0 deletions packages/js-sdk/tests/template/build.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -72,3 +72,20 @@ buildTemplateTest(
await buildTemplate(template)
}
)

buildTemplateTest(
'build template with absolute paths',
async ({ buildTemplate }) => {
const packageTxt = path.resolve(process.cwd(), folderPath, 'test.txt')

const template = Template()
// using base image to avoid re-building ubuntu:22.04 image
.fromBaseImage()
.skipCache()
.copy(packageTxt, 'text.txt', { forceUpload: true })
.copy('../../../../package.json', 'package.json', { forceUpload: true })
.runCmd(['ls -l .', 'cat text.txt', 'cat package.json'])

await buildTemplate(template, {}, defaultBuildLogger())
}
)
16 changes: 13 additions & 3 deletions packages/python-sdk/e2b/template/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
read_dockerignore,
read_gcp_service_account_json,
get_caller_frame,
normalize_copy_source_path,
)
from types import TracebackType

Expand Down Expand Up @@ -65,8 +66,11 @@ def copy(
srcs = [src] if isinstance(src, (str, Path)) else src

for src_item in srcs:
normalized_src, context_path_for_instruction = normalize_copy_source_path(
str(src_item), self._template._file_context_path
)
args = [
str(src_item),
normalized_src,
str(dest),
user or "",
pad_octal(mode) if mode else "",
Expand All @@ -78,6 +82,7 @@ def copy(
"force": force_upload or self._template._force_next_layer,
"forceUpload": force_upload,
"resolveSymlinks": resolve_symlinks,
"contextPath": context_path_for_instruction,
}

self._template._instructions.append(instruction)
Expand Down Expand Up @@ -1275,6 +1280,7 @@ def _instructions_with_hashes(
"force": instruction["force"],
"forceUpload": instruction.get("forceUpload"),
"resolveSymlinks": instruction.get("resolveSymlinks"),
"contextPath": instruction.get("contextPath"),
}

if instruction["type"] == InstructionType.COPY:
Expand All @@ -1289,13 +1295,17 @@ def _instructions_with_hashes(
raise ValueError("Source path and destination path are required")

resolve_symlinks = instruction.get("resolveSymlinks")
context_path_for_instruction = (
instruction.get("contextPath") or self._file_context_path
)

step["filesHash"] = calculate_files_hash(
src,
dest,
self._file_context_path,
context_path_for_instruction,
[
*self._file_ignore_patterns,
*read_dockerignore(self._file_context_path),
*read_dockerignore(context_path_for_instruction),
],
resolve_symlinks
if resolve_symlinks is not None
Expand Down
1 change: 1 addition & 0 deletions packages/python-sdk/e2b/template/types.py
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ class Instruction(TypedDict):
forceUpload: NotRequired[Optional[Literal[True]]]
filesHash: NotRequired[Optional[str]]
resolveSymlinks: NotRequired[Optional[bool]]
contextPath: NotRequired[Optional[str]]


class GenericDockerRegistry(TypedDict):
Expand Down
56 changes: 50 additions & 6 deletions packages/python-sdk/e2b/template/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,48 @@ def normalize_path(path: str) -> str:
return path.replace(os.sep, "/")


def normalize_copy_source_path(src: str, file_context_path: str) -> (str, str):
"""
Normalize a COPY source path into a context-relative pattern and its context.

- Absolute sources: anchor context at '/', keep full path structure
(relpath from '/'; '.' when the source is '/').
- Relative sources: context defaults to file_context_path; if the path escapes
that context (e.g., '../../../foo'), anchor at '/' to preserve structure.
- Always returns POSIX separators for glob/tar friendliness.

:param src: The source path to normalize
:param file_context_path: The context path to use
:return: A tuple containing the normalized source path and the context path
"""
default_context = os.path.abspath(file_context_path)
absolute_src = (
src
if os.path.isabs(src)
else os.path.abspath(os.path.join(default_context, src))
)

if os.path.isabs(src):
context_path_for_instruction = "/"
normalized_src = normalize_path(os.path.relpath(absolute_src, "/")) or "."
return normalized_src, context_path_for_instruction

relative_to_default = os.path.relpath(absolute_src, default_context)
escapes_default = (
relative_to_default == ".."
or relative_to_default.startswith(".." + os.path.sep)
or relative_to_default.startswith("../")
)
context_path_for_instruction = "/" if escapes_default else default_context

normalized_src = (
normalize_path(os.path.relpath(absolute_src, context_path_for_instruction))
or "."
)

return normalized_src, context_path_for_instruction


def get_all_files_in_path(
src: str,
context_path: str,
Expand All @@ -62,18 +104,20 @@ def get_all_files_in_path(
"""
files = set()

# Use glob to find all files/directories matching the pattern under context_path
# Use glob to find all files/directories. Only set root_dir for relative patterns;
# absolute patterns must be matched as-is (particularly on Windows where drives differ).
abs_context_path = os.path.abspath(context_path)
is_abs_src = os.path.isabs(src)
files_glob = glob.glob(
src,
flags=glob.GLOBSTAR,
root_dir=abs_context_path,
root_dir=None if is_abs_src else abs_context_path,
exclude=ignore_patterns,
)

for file in files_glob:
# Join it with abs_context_path to get the absolute path
file_path = os.path.join(abs_context_path, file)
# If glob returned a relative path, anchor it to abs_context_path
file_path = file if is_abs_src else os.path.join(abs_context_path, file)

if os.path.isdir(file_path):
# If it's a directory, add the directory and all entries recursively
Expand All @@ -82,11 +126,11 @@ def get_all_files_in_path(
dir_files = glob.glob(
normalize_path(file) + "/**/*",
flags=glob.GLOBSTAR,
root_dir=abs_context_path,
root_dir=None if is_abs_src else abs_context_path,
exclude=ignore_patterns,
)
for dir_file in dir_files:
dir_file_path = os.path.join(abs_context_path, dir_file)
dir_file_path = dir_file if is_abs_src else os.path.join(abs_context_path, dir_file)
files.add(dir_file_path)
else:
files.add(file_path)
Expand Down
Loading
Loading