1. 成功允运行了Yolo的分类模型,大体了解了使用方式
2. 成功运行了Gemini2摄像头 3. 成功将摄像头与Yolo模型整合运行
This commit is contained in:
commit
454381eb1c
1
.gitignore
vendored
Normal file
1
.gitignore
vendored
Normal file
@ -0,0 +1 @@
|
||||
/.venv/
|
10
.idea/graduation_design.iml
generated
Normal file
10
.idea/graduation_design.iml
generated
Normal file
@ -0,0 +1,10 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<module type="PYTHON_MODULE" version="4">
|
||||
<component name="NewModuleRootManager">
|
||||
<content url="file://$MODULE_DIR$">
|
||||
<excludeFolder url="file://$MODULE_DIR$/.venv" />
|
||||
</content>
|
||||
<orderEntry type="inheritedJdk" />
|
||||
<orderEntry type="sourceFolder" forTests="false" />
|
||||
</component>
|
||||
</module>
|
6
.idea/inspectionProfiles/profiles_settings.xml
generated
Normal file
6
.idea/inspectionProfiles/profiles_settings.xml
generated
Normal file
@ -0,0 +1,6 @@
|
||||
<component name="InspectionProjectProfileManager">
|
||||
<settings>
|
||||
<option name="USE_PROJECT_PROFILE" value="false" />
|
||||
<version value="1.0" />
|
||||
</settings>
|
||||
</component>
|
7
.idea/misc.xml
generated
Normal file
7
.idea/misc.xml
generated
Normal file
@ -0,0 +1,7 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project version="4">
|
||||
<component name="Black">
|
||||
<option name="sdkName" value="Python 3.11 (graduation_design)" />
|
||||
</component>
|
||||
<component name="ProjectRootManager" version="2" project-jdk-name="Python 3.11 (graduation_design)" project-jdk-type="Python SDK" />
|
||||
</project>
|
8
.idea/modules.xml
generated
Normal file
8
.idea/modules.xml
generated
Normal file
@ -0,0 +1,8 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project version="4">
|
||||
<component name="ProjectModuleManager">
|
||||
<modules>
|
||||
<module fileurl="file://$PROJECT_DIR$/.idea/graduation_design.iml" filepath="$PROJECT_DIR$/.idea/graduation_design.iml" />
|
||||
</modules>
|
||||
</component>
|
||||
</project>
|
11
.idea/saveactions_settings.xml
generated
Normal file
11
.idea/saveactions_settings.xml
generated
Normal file
@ -0,0 +1,11 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project version="4">
|
||||
<component name="SaveActionSettings">
|
||||
<option name="actions">
|
||||
<set>
|
||||
<option value="activate" />
|
||||
<option value="reformat" />
|
||||
</set>
|
||||
</option>
|
||||
</component>
|
||||
</project>
|
6
.idea/vcs.xml
generated
Normal file
6
.idea/vcs.xml
generated
Normal file
@ -0,0 +1,6 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project version="4">
|
||||
<component name="VcsDirectoryMappings">
|
||||
<mapping directory="$PROJECT_DIR$" vcs="Git" />
|
||||
</component>
|
||||
</project>
|
139
.idea/workspace.xml
generated
Normal file
139
.idea/workspace.xml
generated
Normal file
@ -0,0 +1,139 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project version="4">
|
||||
<component name="AutoImportSettings">
|
||||
<option name="autoReloadType" value="SELECTIVE" />
|
||||
</component>
|
||||
<component name="ChangeListManager">
|
||||
<list default="true" id="5de6e434-16d8-46ca-a50f-5cce2ce0877f" name="更改" comment="">
|
||||
<change afterPath="$PROJECT_DIR$/.gitignore" afterDir="false" />
|
||||
<change afterPath="$PROJECT_DIR$/.idea/graduation_design.iml" afterDir="false" />
|
||||
<change afterPath="$PROJECT_DIR$/.idea/inspectionProfiles/profiles_settings.xml" afterDir="false" />
|
||||
<change afterPath="$PROJECT_DIR$/.idea/misc.xml" afterDir="false" />
|
||||
<change afterPath="$PROJECT_DIR$/.idea/modules.xml" afterDir="false" />
|
||||
<change afterPath="$PROJECT_DIR$/.idea/saveactions_settings.xml" afterDir="false" />
|
||||
<change afterPath="$PROJECT_DIR$/.idea/vcs.xml" afterDir="false" />
|
||||
<change afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" />
|
||||
<change afterPath="$PROJECT_DIR$/README.md" afterDir="false" />
|
||||
<change afterPath="$PROJECT_DIR$/camera/__init__.py" afterDir="false" />
|
||||
<change afterPath="$PROJECT_DIR$/camera/image_processing.py" afterDir="false" />
|
||||
<change afterPath="$PROJECT_DIR$/camera/orbbec_camera/__init__.py" afterDir="false" />
|
||||
<change afterPath="$PROJECT_DIR$/camera/orbbec_camera/color_viewer.py" afterDir="false" />
|
||||
<change afterPath="$PROJECT_DIR$/camera/utils.py" afterDir="false" />
|
||||
<change afterPath="$PROJECT_DIR$/camera/yolo.py" afterDir="false" />
|
||||
<change afterPath="$PROJECT_DIR$/camera/yolov8n.pt" afterDir="false" />
|
||||
<change afterPath="$PROJECT_DIR$/distance_measurement/__init__.py" afterDir="false" />
|
||||
<change afterPath="$PROJECT_DIR$/distance_measurement/stereo_vision.py" afterDir="false" />
|
||||
<change afterPath="$PROJECT_DIR$/integration_testing/__init__.py" afterDir="false" />
|
||||
<change afterPath="$PROJECT_DIR$/integration_testing/integration_tests.py" afterDir="false" />
|
||||
<change afterPath="$PROJECT_DIR$/main.py" afterDir="false" />
|
||||
<change afterPath="$PROJECT_DIR$/robot_control/__init__.py" afterDir="false" />
|
||||
<change afterPath="$PROJECT_DIR$/robot_control/control.py" afterDir="false" />
|
||||
</list>
|
||||
<option name="SHOW_DIALOG" value="false" />
|
||||
<option name="HIGHLIGHT_CONFLICTS" value="true" />
|
||||
<option name="HIGHLIGHT_NON_ACTIVE_CHANGELIST" value="false" />
|
||||
<option name="LAST_RESOLUTION" value="IGNORE" />
|
||||
</component>
|
||||
<component name="FileTemplateManagerImpl">
|
||||
<option name="RECENT_TEMPLATES">
|
||||
<list>
|
||||
<option value="Python Script" />
|
||||
</list>
|
||||
</option>
|
||||
</component>
|
||||
<component name="Git.Settings">
|
||||
<option name="RECENT_GIT_ROOT_PATH" value="$PROJECT_DIR$" />
|
||||
</component>
|
||||
<component name="MarkdownSettingsMigration">
|
||||
<option name="stateVersion" value="1" />
|
||||
</component>
|
||||
<component name="ProjectColorInfo">{
|
||||
"associatedIndex": 7
|
||||
}</component>
|
||||
<component name="ProjectId" id="2cl92QEZTLSO2Isqs0rCxgPI1yS" />
|
||||
<component name="ProjectLevelVcsManager" settingsEditedManually="true" />
|
||||
<component name="ProjectViewState">
|
||||
<option name="hideEmptyMiddlePackages" value="true" />
|
||||
<option name="showLibraryContents" value="true" />
|
||||
</component>
|
||||
<component name="PropertiesComponent"><![CDATA[{
|
||||
"keyToString": {
|
||||
"Python.color_viewer.executor": "Run",
|
||||
"Python.image_processing.executor": "Run",
|
||||
"Python.yolo.executor": "Run",
|
||||
"RunOnceActivity.OpenProjectViewOnStart": "true",
|
||||
"RunOnceActivity.ShowReadmeOnStart": "true",
|
||||
"SHARE_PROJECT_CONFIGURATION_FILES": "true",
|
||||
"WebServerToolWindowFactoryState": "false",
|
||||
"ignore_missing_gitignore": "true",
|
||||
"last_opened_file_path": "F:/Projects/PyCharmProjects/graduation_design/.venv/Lib/site-packages",
|
||||
"node.js.detected.package.eslint": "true",
|
||||
"node.js.detected.package.tslint": "true",
|
||||
"node.js.selected.package.eslint": "(autodetect)",
|
||||
"node.js.selected.package.tslint": "(autodetect)",
|
||||
"nodejs_package_manager_path": "npm",
|
||||
"settings.editor.selected.configurable": "com.jetbrains.python.configuration.PyActiveSdkModuleConfigurable",
|
||||
"vue.rearranger.settings.migration": "true"
|
||||
}
|
||||
}]]></component>
|
||||
<component name="RecentsManager">
|
||||
<key name="CopyFile.RECENT_KEYS">
|
||||
<recent name="F:\Projects\PyCharmProjects\graduation_design\.venv\Lib\site-packages" />
|
||||
<recent name="F:\Projects\PyCharmProjects\graduation_design\camera\orbbec_camera" />
|
||||
<recent name="F:\Projects\PyCharmProjects\graduation_design\camera" />
|
||||
<recent name="F:\Projects\PyCharmProjects\graduation_design\lib" />
|
||||
<recent name="F:\Projects\PyCharmProjects\graduation_design\integration_testing" />
|
||||
</key>
|
||||
<key name="MoveFile.RECENT_KEYS">
|
||||
<recent name="F:\Projects\PyCharmProjects\graduation_design\camera" />
|
||||
<recent name="F:\Projects\PyCharmProjects\graduation_design\lib" />
|
||||
<recent name="F:\Projects\PyCharmProjects\graduation_design\camera\orbbec_camera" />
|
||||
<recent name="F:\Projects\PyCharmProjects\graduation_design\camera\orbbec_camera\lib" />
|
||||
<recent name="F:\Projects\PyCharmProjects\graduation_design\yolo" />
|
||||
</key>
|
||||
</component>
|
||||
<component name="SharedIndexes">
|
||||
<attachedChunks>
|
||||
<set>
|
||||
<option value="bundled-python-sdk-5a2391486177-2887949eec09-com.jetbrains.pycharm.pro.sharedIndexes.bundled-PY-233.13763.11" />
|
||||
</set>
|
||||
</attachedChunks>
|
||||
</component>
|
||||
<component name="SpellCheckerSettings" RuntimeDictionaries="0" Folders="0" CustomDictionaries="0" DefaultDictionary="应用程序级" UseSingleDictionary="true" transferred="true" />
|
||||
<component name="TaskManager">
|
||||
<task active="true" id="Default" summary="默认任务">
|
||||
<changelist id="5de6e434-16d8-46ca-a50f-5cce2ce0877f" name="更改" comment="" />
|
||||
<created>1708675135052</created>
|
||||
<option name="number" value="Default" />
|
||||
<option name="presentableId" value="Default" />
|
||||
<updated>1708675135052</updated>
|
||||
<workItem from="1708676369490" duration="2293000" />
|
||||
<workItem from="1708772527207" duration="4012000" />
|
||||
<workItem from="1708831408424" duration="3086000" />
|
||||
<workItem from="1708912830842" duration="533000" />
|
||||
<workItem from="1708913406749" duration="4000" />
|
||||
<workItem from="1708913419694" duration="1196000" />
|
||||
<workItem from="1708915341673" duration="14000" />
|
||||
<workItem from="1708915370802" duration="660000" />
|
||||
<workItem from="1708917617291" duration="82000" />
|
||||
<workItem from="1708917707818" duration="36000" />
|
||||
<workItem from="1708917750778" duration="29000" />
|
||||
<workItem from="1708917838563" duration="18000" />
|
||||
<workItem from="1708917889599" duration="5000" />
|
||||
<workItem from="1708917964846" duration="246000" />
|
||||
<workItem from="1708918220728" duration="85000" />
|
||||
<workItem from="1708918314610" duration="999000" />
|
||||
<workItem from="1708925007136" duration="10070000" />
|
||||
<workItem from="1708953749808" duration="5840000" />
|
||||
</task>
|
||||
<servers />
|
||||
</component>
|
||||
<component name="TypeScriptGeneratedFilesManager">
|
||||
<option name="version" value="3" />
|
||||
</component>
|
||||
<component name="com.intellij.coverage.CoverageDataManagerImpl">
|
||||
<SUITE FILE_PATH="coverage/graduation_design$yolo.coverage" NAME="yolo 覆盖结果" MODIFIED="1708959490347" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/camera" />
|
||||
<SUITE FILE_PATH="coverage/graduation_design$image_processing.coverage" NAME="image_processing 覆盖结果" MODIFIED="1708957639566" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/camera" />
|
||||
<SUITE FILE_PATH="coverage/graduation_design$color_viewer.coverage" NAME="color_viewer 覆盖结果" MODIFIED="1708957257852" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/camera/orbbec_camera" />
|
||||
</component>
|
||||
</project>
|
52
README.md
Normal file
52
README.md
Normal file
@ -0,0 +1,52 @@
|
||||
## 1. 配置
|
||||
|
||||
| **配置项** | **选项** |
|
||||
|-----------|------------------|
|
||||
| 🌐 语言 | Python |
|
||||
| 📚 项目类型 | 通用开发,含计算机视觉和机械控制 |
|
||||
| 📖 注释风格 | 描述性 + 内联 |
|
||||
| 🛠️ 代码结构 | 模块化 |
|
||||
| 🚫 错误处理策略 | 强健 + 上下文 |
|
||||
| ⚡ 性能优化等级 | 中等 + 可伸缩性关注 |
|
||||
|
||||
**注意:** lib中关于双目摄像头的库文件需要依照系统自行编译,本项目使用的环境为Windows10
|
||||
|
||||
## 2. 设计
|
||||
|
||||
### 2.1. 系统概述
|
||||
|
||||
- **双目结构光相机系统**: 用于桌面物体的识别和定位。
|
||||
- **雷达或双目测距系统**: 选择双目测距,因为它和相机系统更协同。
|
||||
- **机械臂控制系统**: 根据识别和测距结果,控制机械臂移动物体。
|
||||
|
||||
### 2.2. 主要模块
|
||||
|
||||
1. **图像处理模块**: 使用OpenCV和PyTorch进行图像识别和处理。
|
||||
2. **双目测距模块**: 计算物体与相机的距离。
|
||||
3. **机械臂控制模块**: 根据计算结果控制机械臂。
|
||||
4. **集成与测试模块**: 集成各个模块,并进行测试。
|
||||
|
||||
### 2.3. 数据流
|
||||
|
||||
1. 相机捕获图像 -> 2. 图像处理识别物体 -> 3. 双目测距计算距离 -> 4. 机械臂根据距离和位置移动物体。
|
||||
|
||||
## 3. 项目文件结构
|
||||
|
||||
```
|
||||
plaintext
|
||||
robotic_arm_project/
|
||||
|-- camera/
|
||||
| |-- __init__.py
|
||||
| |-- image_processing.py
|
||||
|-- distance_measurement/
|
||||
| |-- __init__.py
|
||||
| |-- stereo_vision.py
|
||||
|-- arm_control/
|
||||
| |-- __init__.py
|
||||
| |-- control.py
|
||||
|-- integration_testing/
|
||||
| |-- __init__.py
|
||||
| |-- integration_tests.py
|
||||
|-- main.py
|
||||
```
|
||||
|
0
camera/__init__.py
Normal file
0
camera/__init__.py
Normal file
4
camera/image_processing.py
Normal file
4
camera/image_processing.py
Normal file
@ -0,0 +1,4 @@
|
||||
from yolo import process_img
|
||||
|
||||
if __name__ == '__main__':
|
||||
process_img()
|
0
camera/orbbec_camera/__init__.py
Normal file
0
camera/orbbec_camera/__init__.py
Normal file
77
camera/orbbec_camera/color_viewer.py
Normal file
77
camera/orbbec_camera/color_viewer.py
Normal file
@ -0,0 +1,77 @@
|
||||
# 导入所需模块
|
||||
from pyorbbecsdk import Pipeline, FrameSet
|
||||
from pyorbbecsdk import Config
|
||||
from pyorbbecsdk import OBSensorType, OBFormat
|
||||
from pyorbbecsdk import OBError
|
||||
from pyorbbecsdk import VideoStreamProfile
|
||||
import cv2
|
||||
from camera.utils import frame_to_bgr_image
|
||||
|
||||
# ESC键的键值
|
||||
ESC_KEY = 27
|
||||
|
||||
|
||||
def main():
|
||||
# 创建配置对象
|
||||
config = Config()
|
||||
# 创建Pipeline对象
|
||||
pipeline = Pipeline()
|
||||
|
||||
try:
|
||||
# 获取颜色传感器的流配置列表
|
||||
profile_list = pipeline.get_stream_profile_list(OBSensorType.COLOR_SENSOR)
|
||||
try:
|
||||
# 尝试获取指定分辨率、格式和帧率的颜色配置
|
||||
color_profile: VideoStreamProfile = profile_list.get_video_stream_profile(640, 0, OBFormat.RGB, 30)
|
||||
except OBError as e:
|
||||
# 如果出错,打印错误信息并使用默认的颜色配置
|
||||
print(e)
|
||||
color_profile = profile_list.get_default_video_stream_profile()
|
||||
|
||||
# 启用颜色流
|
||||
config.enable_stream(color_profile)
|
||||
except Exception as e:
|
||||
# 如果出现异常,打印错误信息并返回
|
||||
print(e)
|
||||
return
|
||||
|
||||
# 启动Pipeline
|
||||
pipeline.start(config)
|
||||
|
||||
while True:
|
||||
try:
|
||||
# 等待获取帧集,最多等待100毫秒
|
||||
frames: FrameSet = pipeline.wait_for_frames(100)
|
||||
if frames is None:
|
||||
continue
|
||||
|
||||
# 获取颜色帧
|
||||
color_frame = frames.get_color_frame()
|
||||
if color_frame is None:
|
||||
continue
|
||||
|
||||
# 将帧转换为BGR格式的图像
|
||||
color_image = frame_to_bgr_image(color_frame)
|
||||
if color_image is None:
|
||||
print("failed to convert frame to image")
|
||||
continue
|
||||
|
||||
# 在窗口中显示颜色图像
|
||||
cv2.imshow("Color Viewer", color_image)
|
||||
|
||||
# 等待用户按键,每次等待1毫秒
|
||||
key = cv2.waitKey(1)
|
||||
# 如果用户按下 'q' 键或 ESC 键,则退出循环
|
||||
if key == ord('q') or key == ESC_KEY:
|
||||
break
|
||||
except KeyboardInterrupt:
|
||||
# 如果捕获到键盘中断信号,则退出循环
|
||||
break
|
||||
|
||||
# 停止Pipeline
|
||||
pipeline.stop()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
# 当脚本作为主程序运行时,调用main函数
|
||||
main()
|
126
camera/utils.py
Normal file
126
camera/utils.py
Normal file
@ -0,0 +1,126 @@
|
||||
# ******************************************************************************
|
||||
# Copyright (c) 2023 Orbbec 3D Technology, Inc
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http:# www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ******************************************************************************
|
||||
from typing import Union, Any, Optional
|
||||
|
||||
from pyorbbecsdk import FormatConvertFilter, VideoFrame
|
||||
from pyorbbecsdk import OBFormat, OBConvertFormat
|
||||
import cv2
|
||||
import numpy as np
|
||||
|
||||
|
||||
def yuyv_to_bgr(frame: np.ndarray, width: int, height: int) -> np.ndarray:
|
||||
yuyv = frame.reshape((height, width, 2))
|
||||
bgr_image = cv2.cvtColor(yuyv, cv2.COLOR_YUV2BGR_YUY2)
|
||||
return bgr_image
|
||||
|
||||
|
||||
def uyvy_to_bgr(frame: np.ndarray, width: int, height: int) -> np.ndarray:
|
||||
uyvy = frame.reshape((height, width, 2))
|
||||
bgr_image = cv2.cvtColor(uyvy, cv2.COLOR_YUV2BGR_UYVY)
|
||||
return bgr_image
|
||||
|
||||
|
||||
def i420_to_bgr(frame: np.ndarray, width: int, height: int) -> np.ndarray:
|
||||
y = frame[0:height, :]
|
||||
u = frame[height:height + height // 4].reshape(height // 2, width // 2)
|
||||
v = frame[height + height // 4:].reshape(height // 2, width // 2)
|
||||
yuv_image = cv2.merge([y, u, v])
|
||||
bgr_image = cv2.cvtColor(yuv_image, cv2.COLOR_YUV2BGR_I420)
|
||||
return bgr_image
|
||||
|
||||
|
||||
def nv21_to_bgr(frame: np.ndarray, width: int, height: int) -> np.ndarray:
|
||||
y = frame[0:height, :]
|
||||
uv = frame[height:height + height // 2].reshape(height // 2, width)
|
||||
yuv_image = cv2.merge([y, uv])
|
||||
bgr_image = cv2.cvtColor(yuv_image, cv2.COLOR_YUV2BGR_NV21)
|
||||
return bgr_image
|
||||
|
||||
|
||||
def nv12_to_bgr(frame: np.ndarray, width: int, height: int) -> np.ndarray:
|
||||
y = frame[0:height, :]
|
||||
uv = frame[height:height + height // 2].reshape(height // 2, width)
|
||||
yuv_image = cv2.merge([y, uv])
|
||||
bgr_image = cv2.cvtColor(yuv_image, cv2.COLOR_YUV2BGR_NV12)
|
||||
return bgr_image
|
||||
|
||||
|
||||
def determine_convert_format(frame: VideoFrame):
|
||||
if frame.get_format() == OBFormat.I420:
|
||||
return OBConvertFormat.I420_TO_RGB888
|
||||
elif frame.get_format() == OBFormat.MJPG:
|
||||
return OBConvertFormat.MJPG_TO_RGB888
|
||||
elif frame.get_format() == OBFormat.YUYV:
|
||||
return OBConvertFormat.YUYV_TO_RGB888
|
||||
elif frame.get_format() == OBFormat.NV21:
|
||||
return OBConvertFormat.NV21_TO_RGB888
|
||||
elif frame.get_format() == OBFormat.NV12:
|
||||
return OBConvertFormat.NV12_TO_RGB888
|
||||
elif frame.get_format() == OBFormat.UYVY:
|
||||
return OBConvertFormat.UYVY_TO_RGB888
|
||||
else:
|
||||
return None
|
||||
|
||||
|
||||
def frame_to_rgb_frame(frame: VideoFrame) -> Union[Optional[VideoFrame], Any]:
|
||||
if frame.get_format() == OBFormat.RGB:
|
||||
return frame
|
||||
convert_format = determine_convert_format(frame)
|
||||
if convert_format is None:
|
||||
print("Unsupported format")
|
||||
return None
|
||||
print("covert format: {}".format(convert_format))
|
||||
convert_filter = FormatConvertFilter()
|
||||
convert_filter.set_format_convert_format(convert_format)
|
||||
rgb_frame = convert_filter.process(frame)
|
||||
if rgb_frame is None:
|
||||
print("Convert {} to RGB failed".format(frame.get_format()))
|
||||
return rgb_frame
|
||||
|
||||
|
||||
def frame_to_bgr_image(frame: VideoFrame) -> Union[Optional[np.array], Any]:
|
||||
width = frame.get_width()
|
||||
height = frame.get_height()
|
||||
color_format = frame.get_format()
|
||||
data = np.asanyarray(frame.get_data())
|
||||
image = np.zeros((height, width, 3), dtype=np.uint8)
|
||||
if color_format == OBFormat.RGB:
|
||||
image = np.resize(data, (height, width, 3))
|
||||
image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)
|
||||
elif color_format == OBFormat.BGR:
|
||||
image = np.resize(data, (height, width, 3))
|
||||
image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
|
||||
elif color_format == OBFormat.YUYV:
|
||||
image = np.resize(data, (height, width, 2))
|
||||
image = cv2.cvtColor(image, cv2.COLOR_YUV2BGR_YUYV)
|
||||
elif color_format == OBFormat.MJPG:
|
||||
image = cv2.imdecode(data, cv2.IMREAD_COLOR)
|
||||
elif color_format == OBFormat.I420:
|
||||
image = i420_to_bgr(data, width, height)
|
||||
return image
|
||||
elif color_format == OBFormat.NV12:
|
||||
image = nv12_to_bgr(data, width, height)
|
||||
return image
|
||||
elif color_format == OBFormat.NV21:
|
||||
image = nv21_to_bgr(data, width, height)
|
||||
return image
|
||||
elif color_format == OBFormat.UYVY:
|
||||
image = np.resize(data, (height, width, 2))
|
||||
image = cv2.cvtColor(image, cv2.COLOR_YUV2BGR_UYVY)
|
||||
else:
|
||||
print("Unsupported color format: {}".format(color_format))
|
||||
return None
|
||||
return image
|
105
camera/yolo.py
Normal file
105
camera/yolo.py
Normal file
@ -0,0 +1,105 @@
|
||||
import cv2
|
||||
from ultralytics import YOLO
|
||||
from pyorbbecsdk import Pipeline, FrameSet
|
||||
from pyorbbecsdk import Config
|
||||
from pyorbbecsdk import OBSensorType, OBFormat
|
||||
from pyorbbecsdk import OBError
|
||||
from pyorbbecsdk import VideoStreamProfile
|
||||
import numpy as np
|
||||
from utils import frame_to_bgr_image
|
||||
|
||||
|
||||
def load_model():
|
||||
# 载入 YOLOv8 模型
|
||||
model = YOLO('yolov8n.pt')
|
||||
return model
|
||||
|
||||
|
||||
# 参数说明
|
||||
# --img: 输入图片的路径
|
||||
# --is_save_video: 是否保存识别后的视频
|
||||
def process_img(img=None):
|
||||
# 暂时性配置
|
||||
video_path = "output_video.mp4"
|
||||
|
||||
# 加载模型
|
||||
model = load_model()
|
||||
if img is not None:
|
||||
# 对图像运行 YOLOv8 推理
|
||||
results = model(img)
|
||||
|
||||
# 在图像上可视化结果
|
||||
annotated_img = results[0].plot()
|
||||
|
||||
# 显示带有标注的图像
|
||||
cv2.imshow("YOLOv8 推理", annotated_img)
|
||||
cv2.waitKey(0)
|
||||
cv2.destroyAllWindows()
|
||||
else:
|
||||
# 获取本机摄像头内容
|
||||
cap = cv2.VideoCapture(0)
|
||||
|
||||
# 奥比中光摄像头
|
||||
# ESC键的键值
|
||||
ESC_KEY = 27
|
||||
# 创建配置对象
|
||||
config = Config()
|
||||
# 创建Pipeline对象
|
||||
pipeline = Pipeline()
|
||||
|
||||
try:
|
||||
# 获取颜色传感器的流配置列表
|
||||
profile_list = pipeline.get_stream_profile_list(OBSensorType.COLOR_SENSOR)
|
||||
try:
|
||||
# 尝试获取指定分辨率、格式和帧率的颜色配置
|
||||
color_profile: VideoStreamProfile = profile_list.get_video_stream_profile(700, 0, OBFormat.RGB, 30)
|
||||
except OBError as e:
|
||||
# 如果出错,打印错误信息并使用默认的颜色配置
|
||||
print(e)
|
||||
color_profile = profile_list.get_default_video_stream_profile()
|
||||
|
||||
# 启用颜色流
|
||||
config.enable_stream(color_profile)
|
||||
except Exception as e:
|
||||
# 如果出现异常,打印错误信息并返回
|
||||
print(e)
|
||||
return
|
||||
# 启动Pipeline
|
||||
pipeline.start(config)
|
||||
|
||||
while True:
|
||||
try:
|
||||
# 等待获取帧集,最多等待100毫秒
|
||||
frames: FrameSet = pipeline.wait_for_frames(100)
|
||||
if frames is None:
|
||||
continue
|
||||
|
||||
# 获取颜色帧
|
||||
color_frame = frames.get_color_frame()
|
||||
|
||||
if color_frame is None:
|
||||
continue
|
||||
|
||||
# 将帧转换为BGR格式的图像
|
||||
color_image = frame_to_bgr_image(color_frame)
|
||||
|
||||
model.predict(source=color_image, show=True)
|
||||
|
||||
if color_image is None:
|
||||
print("failed to convert frame to image")
|
||||
continue
|
||||
key = cv2.waitKey(1)
|
||||
if key == ord('q') or key == ESC_KEY:
|
||||
break
|
||||
except KeyboardInterrupt:
|
||||
# 如果捕获到键盘中断信号,则退出循环
|
||||
break
|
||||
# 释放视频捕获对象并关闭显示窗口
|
||||
cap.release()
|
||||
cv2.destroyAllWindows()
|
||||
# 停止Pipeline
|
||||
pipeline.stop()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
process_img()
|
BIN
camera/yolov8n.pt
Normal file
BIN
camera/yolov8n.pt
Normal file
Binary file not shown.
0
distance_measurement/__init__.py
Normal file
0
distance_measurement/__init__.py
Normal file
22
distance_measurement/stereo_vision.py
Normal file
22
distance_measurement/stereo_vision.py
Normal file
@ -0,0 +1,22 @@
|
||||
import numpy as np
|
||||
import cv2
|
||||
|
||||
|
||||
# 双目测距模块
|
||||
|
||||
# 双目相机校准
|
||||
def stereo_calibration(left_camera_params, right_camera_params):
|
||||
# TODO: 实现双目相机的校准
|
||||
pass
|
||||
|
||||
|
||||
# 计算视差
|
||||
def compute_disparity(left_image, right_image):
|
||||
# TODO: 计算左右图像的视差
|
||||
pass
|
||||
|
||||
|
||||
# 计算距离
|
||||
def measure_distance(disparity_map, calibration_data):
|
||||
# TODO: 使用视差图和校准数据计算距离
|
||||
pass
|
0
integration_testing/__init__.py
Normal file
0
integration_testing/__init__.py
Normal file
0
integration_testing/integration_tests.py
Normal file
0
integration_testing/integration_tests.py
Normal file
12
main.py
Normal file
12
main.py
Normal file
@ -0,0 +1,12 @@
|
||||
import cv2
|
||||
import torch
|
||||
|
||||
|
||||
def process_image(image):
|
||||
# todo:使用OpenCV和Torch处理图像,识别物体
|
||||
pass
|
||||
|
||||
|
||||
def get_object_coordinates(image):
|
||||
# todo:返回识别物体的坐标
|
||||
pass
|
0
robot_control/__init__.py
Normal file
0
robot_control/__init__.py
Normal file
26
robot_control/control.py
Normal file
26
robot_control/control.py
Normal file
@ -0,0 +1,26 @@
|
||||
# TODO: 导入机械臂控制相关的库(根据实际使用的SDK)
|
||||
|
||||
# 机械臂控制
|
||||
def initialize_arm_control():
|
||||
# TODO: 初始化机械臂控制
|
||||
pass
|
||||
|
||||
|
||||
def move_arm_to_position(x, y, z):
|
||||
# TODO: 移动机械臂到指定位置
|
||||
pass
|
||||
|
||||
|
||||
def grab_object():
|
||||
# TODO: 控制机械臂抓取物体
|
||||
pass
|
||||
|
||||
|
||||
def release_object():
|
||||
# TODO: 控制机械臂释放物体
|
||||
pass
|
||||
|
||||
|
||||
def move_object_to_location(object_coords, target_coords):
|
||||
# TODO: 将物体从当前位置移动到目标位置
|
||||
pass
|
Loading…
Reference in New Issue
Block a user