Hello,

I'm trying to make an app that detects objects in still images, using SwiftUI, PhotosUI, Vision and an object detectior model trained with Create ML.

This is a simplified version of the code I wrote for the view:

import SwiftUI
import PhotosUI

struct ExampleView: View {
    @StateObject private var handler = ExampleVisionHandler()
    @State private var selectedItem: PhotosPickerItem?
    @State private var selectedImage = UIImage(named: "image")!

    var body: some View 
        VStack {
            PhotosPicker("Select an image", selection: $selectedItem, matching: .images)

            Image(uiImage: selectedImage)
                .resizable()
                .scaledToFill()

            Button("Scan") {
                handler.scan(selectedImage.cgImage)
            }
        }
        .onChange(of: selectedItem) { _ in
            Task {
                if let data = try? await selectedItem?.loadTransferable(type: Data.self) {
                    if let uiImage = UIImage(data: data) {
                        selectedImage = uiImage
                        return
                    }
                }

                print("Failed loading image into selectedImage.")
            }
        }
    }
}

which uses this ExampleVisionHandler:

import Vision

class ExampleVisionHandler: ObservableObject {
    var image: CGImage!

    private var requests: [VNRequest] = []
    private var results: [VNRecognizedObjectObservation] = []

    func scan(_ image: CGImage?) {
        if let image {
            self.image = image
            performRequests()
        }
    }

    private func performRequests() {
        let imageRequestHandler = VNImageRequestHandler(cgImage: self.image)
        do {
            try imageRequestHandler.perform(self.requests)
        } catch {
            print(error)
        }
    }

    @discardableResult
    func setupVision() -> NSError? {
        let error: NSError! = nil

        guard let modelURL = Bundle.main.url(forResource: "ObjectDetector 1", withExtension: "mlmodelc") else {
            return NSError(domain: "VisionHandler", code: -1, userInfo: [NSLocalizedDescriptionKey: "Model file is missing"])
        }
        do {
            let visionModel = try VNCoreMLModel(for: MLModel(contentsOf: modelURL))
            let objectRecognition = VNCoreMLRequest(model: visionModel, completionHandler: { (request, error) in
                print("Handling results...")
                guard let results = request.results as? [VNRecognizedObjectObservation] else { print("No results array"); return }
                guard results.isEmpty == false else { print("Results array is empty, was handling \(self.requests.count) requests"); return }

                print("Top results: \(results.first!.labels) : \(results.first!.confidence)")
                self.results = results
            })
            self.requests = [objectRecognition]
        } catch let error as NSError {
            print("Model loading went wrong: \(error)")
        }

        return error
    }

    init() {
        setupVision()
    }
}

If I don't select an image and I just scan the default value of selectedImage (UIImage(named: "image")!), which is saved in the assets, a results array containing several recognized objects will be returned. But if I select exactly the same image from my photo library, the results array will be empty.

Does anyone know what might cause this and/or how to solve it? Thank you in advance!

Did you figure this out?