Skip unsupported file types during source crawl

internal/controller/sourcecrawler_controller.go

@@ -106,6 +106,7 @@ func (r *SourceCrawlerReconciler) Reconcile(ctx context.Context, req ctrl.Reques
 	outputDir := unstructured.StagePath(parentPipeline, sourceCrawlerCR.Spec.StageName)
 
 	var source unstructured.DataSource
+	unsupportedFiles := &unstructured.UnsupportedFiles{}
 	switch sourceCrawlerConfig.Type {
 	case operatorv1alpha1.TypeS3:
 		sourceAWSConfig, err := controllerutils.AWSConfigFromSecret(ctx, r.Client, sourceCrawlerCR.Spec.SecretRef, sourceCrawlerCR.Namespace, "SOURCE_S3_")
@@ -117,14 +118,15 @@ func (r *SourceCrawlerReconciler) Reconcile(ctx context.Context, req ctrl.Reques
 			return ctrl.Result{}, r.handleError(ctx, sourceCrawlerCR, fmt.Errorf("failed to create source S3 client: %w", err))
 		}
 		source = &unstructured.S3BucketSource{
-			S3Client:  sourceS3Client,
-			Bucket:    sourceCrawlerConfig.S3Config.Bucket,
-			Prefix:    sourceCrawlerConfig.S3Config.Prefix,
-			OutputDir: outputDir,
+			S3Client:         sourceS3Client,
+			Bucket:           sourceCrawlerConfig.S3Config.Bucket,
+			Prefix:           sourceCrawlerConfig.S3Config.Prefix,
+			OutputDir:        outputDir,
+			UnsupportedFiles: unsupportedFiles,
 		}
 
 	case operatorv1alpha1.TypeGoogleDrive:
-		gdriveSource, err := r.buildGDriveSource(ctx, sourceCrawlerCR, sourceCrawlerConfig.GoogleDriveConfig, outputDir)
+		gdriveSource, err := r.buildGDriveSource(ctx, sourceCrawlerCR, sourceCrawlerConfig.GoogleDriveConfig, outputDir, unsupportedFiles)
 		if err != nil {
 			return ctrl.Result{}, r.handleError(ctx, sourceCrawlerCR, err)
 		}
@@ -156,6 +158,11 @@ func (r *SourceCrawlerReconciler) Reconcile(ctx context.Context, req ctrl.Reques
 	logger.Info("successfully stored files to filestore", "count", len(storedFiles))
 
 	successMessage := fmt.Sprintf("successfully reconciled source crawler: %s", sourceCrawlerCR.Name)
+	if skipped := unsupportedFiles.List(); len(skipped) > 0 {
+		skippedMessage := fmt.Sprintf("; skipped %d unsupported file(s): %s",
+			len(skipped), formatSkippedFiles(skipped, 10))
+		successMessage += skippedMessage
+	}
 	if err := controllerutils.StatusPatch(ctx, r.Client, sourceCrawlerCR, func() {
 		sourceCrawlerCR.Status.FilesProcessed += int64(len(storedFiles))
 		sourceCrawlerCR.Status.GDriveStatus = gdriveStatus
@@ -212,6 +219,7 @@ func (r *SourceCrawlerReconciler) buildGDriveSource(
 	sourceCrawlerCR *operatorv1alpha1.SourceCrawler,
 	gdriveConfig *operatorv1alpha1.GoogleDriveConfig,
 	outputDir string,
+	unsupportedFiles *unstructured.UnsupportedFiles,
 ) (*unstructured.GDriveSource, error) {
 	if gdriveConfig == nil {
 		return nil, errors.New("gdriveConfig is required when source type is gdrive")
@@ -283,6 +291,7 @@ func (r *SourceCrawlerReconciler) buildGDriveSource(
 		ConcurrentFolders:   concurrentFolders,
 		ConcurrentDownloads: concurrentDownloads,
 		OutputDir:           outputDir,
+		UnsupportedFiles:    unsupportedFiles,
 	}, nil
 }
 
@@ -339,6 +348,13 @@ func (r *SourceCrawlerReconciler) handleError(ctx context.Context, sourceCrawler
 	return reconcileErr
 }
 
+func formatSkippedFiles(files []string, limit int) string {
+	if len(files) <= limit {
+		return strings.Join(files, ", ")
+	}
+	return fmt.Sprintf("%s, and %d more", strings.Join(files[:limit], ", "), len(files)-limit)
+}
+
 // findDependents maps a changed pipeline stage back to the SourceCrawlers that depend on it.
 //
 // Given a SourceCrawler CR like:

pkg/unstructured/crawl_result.go

@@ -0,0 +1,85 @@
+/*
+Copyright 2026.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package unstructured
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"path"
+	"strings"
+	"time"
+
+	"github.com/redhat-data-and-ai/unstructured-data-controller/pkg/filestore"
+)
+
+const (
+	CrawlStatusSuccessful = "successful"
+	CrawlStatusSkipped    = "skipped"
+	CrawlStatusError      = "error"
+	CrawlMetadataDir      = "crawl_metadata"
+	extensionNone         = "(none)"
+)
+
+// CrawlResult is one JSON object per discovered file under stages/<crawl>/catalog/.
+// The same catalog/<fileId>.json path is upserted on each reconcile.
+type CrawlResult struct {
+	FileID     string `json:"file_id"`
+	FileName   string `json:"file_name"`
+	SourcePath string `json:"source_path,omitempty"`
+	FileURL    string `json:"file_url,omitempty"`
+	MediaType  string `json:"media_type,omitempty"`
+	Extension  string `json:"extension"` // without leading dot, e.g. "pdf"
+	Status     string `json:"status"`
+	Reason     string `json:"reason,omitempty"`
+	SourceType string `json:"source_type"`
+	CrawledAt  string `json:"crawled_at"`
+}
+
+func CrawlCatalogPath(outputDir, fileID string) string {
+	return path.Join(outputDir, CrawlMetadataDir, fileID+".json")
+}
+
+// GDriveFileURL returns the standard Google Drive web link for a file ID.
+func GDriveFileURL(fileID string) string {
+	return fmt.Sprintf("https://drive.google.com/file/d/%s/view", fileID)
+}
+
+// catalogExtension returns an extension without a leading dot (e.g. "pdf").
+func catalogExtension(ext string) string {
+	ext = strings.ToLower(strings.TrimSpace(ext))
+	if ext == "" || ext == extensionNone {
+		return extensionNone
+	}
+	return strings.TrimPrefix(ext, ".")
+}
+
+func storeCrawlResult(ctx context.Context, fs *filestore.FileStore, outputDir string, r CrawlResult) error {
+	if r.CrawledAt == "" {
+		r.CrawledAt = time.Now().UTC().Format(time.RFC3339)
+	}
+	if r.Extension == "" {
+		r.Extension = catalogExtension(FileExtension(r.FileName))
+	} else {
+		r.Extension = catalogExtension(r.Extension)
+	}
+	data, err := json.Marshal(r)
+	if err != nil {
+		return err
+	}
+	return fs.Store(ctx, CrawlCatalogPath(outputDir, r.FileID), data)
+}

pkg/unstructured/file_types.go

@@ -0,0 +1,53 @@
+/*
+Copyright 2026.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package unstructured
+
+import (
+	"path"
+	"strings"
+)
+
+// SupportedFileExtensions mirrors Docling-compatible document extensions, excluding image.
+// Supported: docx, doc, pptx, html, pdf, asciidoc, md/txt, csv, xlsx.
+var SupportedFileExtensions = map[string]bool{
+	".docx":     true,
+	".doc":      true, // legacy Word (GDrive exports as PDF; S3 needs LibreOffice/Docling support)
+	".pptx":     true,
+	".html":     true,
+	".htm":      true,
+	".pdf":      true,
+	".adoc":     true,
+	".asciidoc": true,
+	".md":       true,
+	".markdown": true,
+	".txt":      true, // Docling treats as MD
+	".text":     true, // Docling treats as MD
+	".csv":      true,
+	".xlsx":     true,
+}
+
+func IsSupportedFileType(fileName string) bool {
+	return SupportedFileExtensions[strings.ToLower(path.Ext(fileName))]
+}
+
+func FileExtension(fileName string) string {
+	ext := strings.ToLower(path.Ext(fileName))
+	if ext == "" {
+		return extensionNone
+	}
+	return ext
+}