Open
Description
I followed the guide in https://apple.github.io/coremltools/docs-guides/source/convert-openelm.html, trying to convert llm-like model to CoreML format, but encountered a bug with this line of code
mlmodel = ct.convert(exported_program)
The bug:
When both 'convert_to' and 'minimum_deployment_target' not specified, 'convert_to' is set to "mlprogram" and 'minimum_deployment_target' is set to ct.target.iOS15 (which is same as ct.target.macOS12). Note: the model will not run on systems older than iOS15/macOS12/watchOS8/tvOS15. In order to make your model run on older system, please set the 'minimum_deployment_target' to iOS14/iOS13. Details please see the link: https://apple.github.io/coremltools/docs-guides/source/target-conversion-formats.html
---------------------------------------------------------------------------
ValueError Traceback (most recent call last)
Cell In[30], line 1
----> 1 mlmodel = ct.convert(
2 exported_program,
3 # convert_to="neuralnetwork",
4 )
File ~/anaconda3/envs/env-coreml/lib/python3.12/site-packages/coremltools/converters/_converters_entry.py:635, in convert(model, source, inputs, outputs, classifier_config, minimum_deployment_target, convert_to, compute_precision, skip_model_load, compute_units, package_dir, debug, pass_pipeline, states)
632 if len(states) > 0 and exact_source != "pytorch":
633 raise ValueError("'states' can only be passed with pytorch source model.")
--> 635 mlmodel = mil_convert(
636 model,
637 convert_from=exact_source,
638 convert_to=exact_target,
639 inputs=inputs,
640 outputs=outputs_as_tensor_or_image_types, # None or list[ct.ImageType/ct.TensorType]
641 classifier_config=classifier_config,
642 skip_model_load=skip_model_load,
643 compute_units=compute_units,
644 package_dir=package_dir,
645 debug=debug,
646 specification_version=specification_version,
647 main_pipeline=pass_pipeline,
648 use_default_fp16_io=use_default_fp16_io,
649 states=states,
650 )
652 if exact_target == "mlprogram" and mlmodel._input_has_infinite_upper_bound():
653 raise ValueError(
654 "For mlprogram, inputs with infinite upper_bound is not allowed. Please set upper_bound"
655 ' to a positive value in "RangeDim()" for the "inputs" param in ct.convert().'
656 )
File ~/anaconda3/envs/env-coreml/lib/python3.12/site-packages/coremltools/converters/mil/converter.py:186, in mil_convert(model, convert_from, convert_to, compute_units, **kwargs)
147 @_profile
148 def mil_convert(
149 model,
(...) 153 **kwargs
154 ):
155 """
156 Convert model from a specified frontend `convert_from` to a specified
157 converter backend `convert_to`.
(...) 184 See `coremltools.converters.convert`
185 """
--> 186 return _mil_convert(
187 model,
188 convert_from,
189 convert_to,
190 ConverterRegistry,
191 ct.models.MLModel,
192 compute_units,
193 **kwargs,
194 )
File ~/anaconda3/envs/env-coreml/lib/python3.12/site-packages/coremltools/converters/mil/converter.py:218, in _mil_convert(model, convert_from, convert_to, registry, modelClass, compute_units, **kwargs)
215 weights_dir = _tempfile.TemporaryDirectory()
216 kwargs["weights_dir"] = weights_dir.name
--> 218 proto, mil_program = mil_convert_to_proto(
219 model,
220 convert_from,
221 convert_to,
222 registry,
223 **kwargs
224 )
226 _reset_conversion_state()
228 if convert_to == 'milinternal':
File ~/anaconda3/envs/env-coreml/lib/python3.12/site-packages/coremltools/converters/mil/converter.py:294, in mil_convert_to_proto(model, convert_from, convert_to, converter_registry, main_pipeline, **kwargs)
289 frontend_pipeline, backend_pipeline = _construct_other_pipelines(
290 main_pipeline, convert_from, convert_to
291 )
293 frontend_converter = frontend_converter_type()
--> 294 prog = frontend_converter(model, **kwargs)
295 PassPipelineManager.apply_pipeline(prog, frontend_pipeline)
297 PassPipelineManager.apply_pipeline(prog, main_pipeline)
File ~/anaconda3/envs/env-coreml/lib/python3.12/site-packages/coremltools/converters/mil/converter.py:106, in TorchFrontend.__call__(self, *args, **kwargs)
103 def __call__(self, *args, **kwargs):
104 from .frontend.torch.load import load
--> 106 return load(*args, **kwargs)
File ~/anaconda3/envs/env-coreml/lib/python3.12/site-packages/coremltools/converters/mil/frontend/torch/load.py:78, in load(spec, inputs, specification_version, debug, outputs, cut_at_symbols, use_default_fp16_io, states, **kwargs)
75 else:
76 model = _torchscript_from_spec(spec)
---> 78 converter = TorchConverter(
79 model,
80 inputs,
81 outputs,
82 cut_at_symbols,
83 specification_version,
84 use_default_fp16_io,
85 states,
86 )
88 return _perform_torch_convert(converter, debug)
File ~/anaconda3/envs/env-coreml/lib/python3.12/site-packages/coremltools/converters/mil/frontend/torch/converter.py:626, in TorchConverter.__init__(self, loaded_model, inputs, outputs, cut_at_symbols, opset_version, use_default_fp16_io, states)
621 raise NotImplementedError(
622 "Conversion for models with only ATEN or EDGE dialect is supported/tested. "
623 f"Provided Dialect: {loaded_model.dialect}"
624 )
625 self.context = TranscriptionContext(frontend=frontend)
--> 626 self.graph = InternalTorchIRGraph.from_exir(
627 exir=loaded_model, cut_at_symbols=cut_at_symbols
628 )
630 # finalize inputs after internal graph gets settled
631 self.inputs = self._match_user_exir_inputs(inputs)
File ~/anaconda3/envs/env-coreml/lib/python3.12/site-packages/coremltools/converters/mil/frontend/torch/internal_graph.py:590, in InternalTorchIRGraph.from_exir(cls, exir, cut_at_symbols)
588 for node in exported_program.graph_module.graph.nodes:
589 if node.op == "call_function":
--> 590 nodes.append(InternalTorchIRNode.from_exir_node(node=node))
591 elif node.op == "get_attr":
592 name = node.target
File ~/anaconda3/envs/env-coreml/lib/python3.12/site-packages/coremltools/converters/mil/frontend/torch/internal_graph.py:289, in InternalTorchIRNode.from_exir_node(cls, node)
287 kind = sanitize_op_kind(kind)
288 if kind not in _TORCH_OPS_REGISTRY:
--> 289 raise ValueError(f"Unsupported fx node {str(node)}, kind {kind}")
291 inputs = get_arguments(node.args)
292 outputs = [node.name]
ValueError: Unsupported fx node bitwise_or, kind bitwise_or
And I also want to know when we want to run llm on apple device, do we usually convert the llm to CoreML format or there is another way?