diff --git a/.gitignore b/.gitignore index 6210b861..f833a7f0 100644 --- a/.gitignore +++ b/.gitignore @@ -15,7 +15,7 @@ *.out # Dependency directories (remove the comment below to include it) -# vendor/ +/vendor # Go workspace file go.work diff --git a/pkg/backend/build/builder.go b/pkg/backend/build/builder.go index d971589a..6ed2b534 100644 --- a/pkg/backend/build/builder.go +++ b/pkg/backend/build/builder.go @@ -17,6 +17,7 @@ package build import ( + "archive/tar" "bytes" "context" "encoding/json" @@ -26,6 +27,7 @@ import ( "os" "path/filepath" "strconv" + "strings" "sync" "syscall" "time" @@ -123,12 +125,56 @@ type abstractBuilder struct { } func (ab *abstractBuilder) BuildLayer(ctx context.Context, mediaType, workDir, path string, hooks hooks.Hooks) (ocispec.Descriptor, error) { + logrus.Debugf("BuildLayer: mediaType %s, workDir %s, path %s", mediaType, workDir, path) info, err := os.Stat(path) if err != nil { return ocispec.Descriptor{}, fmt.Errorf("failed to get file info: %w", err) } if info.IsDir() { + lInfo, err := os.Lstat(path) + if err != nil { + return ocispec.Descriptor{}, fmt.Errorf("failed to get file info: %w", err) + } + isSymlink := lInfo.Mode()&os.ModeSymlink != 0 + if isSymlink { + // 对于符号链接，获取其指向的目标信息 + dstLinkPath, err := os.Readlink(path) + if err != nil { + return ocispec.Descriptor{}, fmt.Errorf("failed to read symlink: %w", err) + } + + workDirPath, err := filepath.Abs(workDir) + if err != nil { + return ocispec.Descriptor{}, fmt.Errorf("failed to get absolute path of workDir: %w", err) + } + + // Gets the relative path of the file as annotation. + //nolint:typecheck + relPath, err := filepath.Rel(workDirPath, path) + if err != nil { + return ocispec.Descriptor{}, fmt.Errorf("failed to get relative path: %w", err) + } + logrus.Debugf("builder: starting build layer for synlink %s, target path %s", relPath, dstLinkPath) + + // build layer for the target of symbolic link. + reader := strings.NewReader("") + hash := sha256.New() + size, err := io.Copy(hash, reader) + if err != nil { + return ocispec.Descriptor{}, fmt.Errorf("failed to copy content to hash: %w", err) + } + digest := fmt.Sprintf("sha256:%x", hash.Sum(nil)) + desc, err := ab.strategy.OutputLayer(ctx, mediaType, relPath, digest, size, reader, hooks) + if err != nil { + return desc, err + } + // Add file metadata to descriptor. + if err := addFileMetadata(&desc, path, relPath); err != nil { + return desc, err + } + return desc, nil + } return ocispec.Descriptor{}, fmt.Errorf("%s is a directory and not supported yet", path) } @@ -432,11 +478,27 @@ func getFileMetadata(path string) (modelspec.FileMetadata, error) { // Set Typeflag. switch { case info.Mode().IsRegular(): - metadata.Typeflag = 0 // Regular file + metadata.Typeflag = tar.TypeReg // Regular file case info.Mode().IsDir(): - metadata.Typeflag = 5 // Directory + metadata.Typeflag = tar.TypeDir // Directory + lInfo, err := os.Lstat(path) + if err != nil { + return metadata, fmt.Errorf("failed to get file info: %w", err) + } + isSymlink := lInfo.Mode()&os.ModeSymlink != 0 + if isSymlink { + // 对于符号链接，获取其指向的目标信息 + dstLinkPath, err := os.Readlink(path) + if err != nil { + return metadata, fmt.Errorf("failed to read symlink: %w", err) + } + logrus.Debugf("builder: symlink detected for file %s -> %s", path, dstLinkPath) + metadata.Typeflag = tar.TypeSymlink // Symlink + metadata.DstLinkPath = dstLinkPath + } + case info.Mode()&os.ModeSymlink != 0: - metadata.Typeflag = 2 // Symlink + metadata.Typeflag = tar.TypeSymlink // Symlink default: return metadata, errors.New("unknown file typeflag") } diff --git a/pkg/backend/extract.go b/pkg/backend/extract.go index fc36cd51..b33477da 100644 --- a/pkg/backend/extract.go +++ b/pkg/backend/extract.go @@ -17,12 +17,15 @@ package backend import ( + "archive/tar" "bufio" "context" "encoding/json" "errors" "fmt" "io" + "os" + "path/filepath" legacymodelspec "github.com/dragonflyoss/model-spec/specs-go/v1" modelspec "github.com/modelpack/model-spec/specs-go/v1" @@ -114,22 +117,57 @@ func exportModelArtifact(ctx context.Context, store storage.Storage, manifest oc // extractLayer extracts the layer to the output directory. func extractLayer(desc ocispec.Descriptor, outputDir string, reader io.Reader) error { - var filepath string + var filePath string if desc.Annotations != nil { if desc.Annotations[modelspec.AnnotationFilepath] != "" { - filepath = desc.Annotations[modelspec.AnnotationFilepath] + filePath = desc.Annotations[modelspec.AnnotationFilepath] } else { - filepath = desc.Annotations[legacymodelspec.AnnotationFilepath] + filePath = desc.Annotations[legacymodelspec.AnnotationFilepath] } } + // load symlink from annotation if exists + var fileMetadata *modelspec.FileMetadata + // Try to retrieve the file metadata from annotation for raw file. + if desc.Annotations != nil { + fileMetadataStr := desc.Annotations[modelspec.AnnotationFileMetadata] + if fileMetadataStr == "" { + fileMetadataStr = desc.Annotations[legacymodelspec.AnnotationFileMetadata] + } + + if fileMetadataStr != "" { + if err := json.Unmarshal([]byte(fileMetadataStr), &fileMetadata); err != nil { + return err + } + } + } + if fileMetadata != nil { + if fileMetadata.Typeflag == tar.TypeSymlink && len(fileMetadata.DstLinkPath) > 0 { + // handle file metadata + + dstFullPath := filepath.Join(outputDir, fileMetadata.DstLinkPath) + if err := os.MkdirAll(dstFullPath, 0755); err != nil && !os.IsExist(err) { + return fmt.Errorf("failed to create directory %s: %w", dstFullPath, err) + } + fullPath := filepath.Join(outputDir, filePath) + if err := os.Remove(fullPath); err != nil && !os.IsNotExist(err) { + return fmt.Errorf("failed to remove file %s: %w", fullPath, err) + } + if err := os.Symlink(fileMetadata.DstLinkPath, fullPath); err != nil { + return fmt.Errorf("failed to create symlink from %s to %s: %w", fullPath, dstFullPath, err) + } + logrus.Debugf("extract: created symlink from %s to %s", fullPath, dstFullPath) + return nil + } + } + codec, err := pkgcodec.New(pkgcodec.TypeFromMediaType(desc.MediaType)) if err != nil { return fmt.Errorf("failed to create codec for media type %s: %w", desc.MediaType, err) } - if err := codec.Decode(outputDir, filepath, reader, desc); err != nil { + if err := codec.Decode(outputDir, filePath, reader, desc); err != nil { if errors.Is(err, pkgcodec.ErrAlreadyUpToDate) { return err }