1. 成功允运行了Yolo的分类模型,大体了解了使用方式

2. 成功运行了Gemini2摄像头
3. 成功将摄像头与Yolo模型整合运行
This commit is contained in:
KYOSG 2024-02-26 23:04:32 +08:00
commit 454381eb1c
23 changed files with 612 additions and 0 deletions

1
.gitignore vendored Normal file
View File

@ -0,0 +1 @@
/.venv/

10
.idea/graduation_design.iml generated Normal file
View File

@ -0,0 +1,10 @@
<?xml version="1.0" encoding="UTF-8"?>
<module type="PYTHON_MODULE" version="4">
<component name="NewModuleRootManager">
<content url="file://$MODULE_DIR$">
<excludeFolder url="file://$MODULE_DIR$/.venv" />
</content>
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
</component>
</module>

View File

@ -0,0 +1,6 @@
<component name="InspectionProjectProfileManager">
<settings>
<option name="USE_PROJECT_PROFILE" value="false" />
<version value="1.0" />
</settings>
</component>

7
.idea/misc.xml generated Normal file
View File

@ -0,0 +1,7 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="Black">
<option name="sdkName" value="Python 3.11 (graduation_design)" />
</component>
<component name="ProjectRootManager" version="2" project-jdk-name="Python 3.11 (graduation_design)" project-jdk-type="Python SDK" />
</project>

8
.idea/modules.xml generated Normal file
View File

@ -0,0 +1,8 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ProjectModuleManager">
<modules>
<module fileurl="file://$PROJECT_DIR$/.idea/graduation_design.iml" filepath="$PROJECT_DIR$/.idea/graduation_design.iml" />
</modules>
</component>
</project>

11
.idea/saveactions_settings.xml generated Normal file
View File

@ -0,0 +1,11 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="SaveActionSettings">
<option name="actions">
<set>
<option value="activate" />
<option value="reformat" />
</set>
</option>
</component>
</project>

6
.idea/vcs.xml generated Normal file
View File

@ -0,0 +1,6 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="VcsDirectoryMappings">
<mapping directory="$PROJECT_DIR$" vcs="Git" />
</component>
</project>

139
.idea/workspace.xml generated Normal file
View File

@ -0,0 +1,139 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="AutoImportSettings">
<option name="autoReloadType" value="SELECTIVE" />
</component>
<component name="ChangeListManager">
<list default="true" id="5de6e434-16d8-46ca-a50f-5cce2ce0877f" name="更改" comment="">
<change afterPath="$PROJECT_DIR$/.gitignore" afterDir="false" />
<change afterPath="$PROJECT_DIR$/.idea/graduation_design.iml" afterDir="false" />
<change afterPath="$PROJECT_DIR$/.idea/inspectionProfiles/profiles_settings.xml" afterDir="false" />
<change afterPath="$PROJECT_DIR$/.idea/misc.xml" afterDir="false" />
<change afterPath="$PROJECT_DIR$/.idea/modules.xml" afterDir="false" />
<change afterPath="$PROJECT_DIR$/.idea/saveactions_settings.xml" afterDir="false" />
<change afterPath="$PROJECT_DIR$/.idea/vcs.xml" afterDir="false" />
<change afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" />
<change afterPath="$PROJECT_DIR$/README.md" afterDir="false" />
<change afterPath="$PROJECT_DIR$/camera/__init__.py" afterDir="false" />
<change afterPath="$PROJECT_DIR$/camera/image_processing.py" afterDir="false" />
<change afterPath="$PROJECT_DIR$/camera/orbbec_camera/__init__.py" afterDir="false" />
<change afterPath="$PROJECT_DIR$/camera/orbbec_camera/color_viewer.py" afterDir="false" />
<change afterPath="$PROJECT_DIR$/camera/utils.py" afterDir="false" />
<change afterPath="$PROJECT_DIR$/camera/yolo.py" afterDir="false" />
<change afterPath="$PROJECT_DIR$/camera/yolov8n.pt" afterDir="false" />
<change afterPath="$PROJECT_DIR$/distance_measurement/__init__.py" afterDir="false" />
<change afterPath="$PROJECT_DIR$/distance_measurement/stereo_vision.py" afterDir="false" />
<change afterPath="$PROJECT_DIR$/integration_testing/__init__.py" afterDir="false" />
<change afterPath="$PROJECT_DIR$/integration_testing/integration_tests.py" afterDir="false" />
<change afterPath="$PROJECT_DIR$/main.py" afterDir="false" />
<change afterPath="$PROJECT_DIR$/robot_control/__init__.py" afterDir="false" />
<change afterPath="$PROJECT_DIR$/robot_control/control.py" afterDir="false" />
</list>
<option name="SHOW_DIALOG" value="false" />
<option name="HIGHLIGHT_CONFLICTS" value="true" />
<option name="HIGHLIGHT_NON_ACTIVE_CHANGELIST" value="false" />
<option name="LAST_RESOLUTION" value="IGNORE" />
</component>
<component name="FileTemplateManagerImpl">
<option name="RECENT_TEMPLATES">
<list>
<option value="Python Script" />
</list>
</option>
</component>
<component name="Git.Settings">
<option name="RECENT_GIT_ROOT_PATH" value="$PROJECT_DIR$" />
</component>
<component name="MarkdownSettingsMigration">
<option name="stateVersion" value="1" />
</component>
<component name="ProjectColorInfo">{
&quot;associatedIndex&quot;: 7
}</component>
<component name="ProjectId" id="2cl92QEZTLSO2Isqs0rCxgPI1yS" />
<component name="ProjectLevelVcsManager" settingsEditedManually="true" />
<component name="ProjectViewState">
<option name="hideEmptyMiddlePackages" value="true" />
<option name="showLibraryContents" value="true" />
</component>
<component name="PropertiesComponent"><![CDATA[{
"keyToString": {
"Python.color_viewer.executor": "Run",
"Python.image_processing.executor": "Run",
"Python.yolo.executor": "Run",
"RunOnceActivity.OpenProjectViewOnStart": "true",
"RunOnceActivity.ShowReadmeOnStart": "true",
"SHARE_PROJECT_CONFIGURATION_FILES": "true",
"WebServerToolWindowFactoryState": "false",
"ignore_missing_gitignore": "true",
"last_opened_file_path": "F:/Projects/PyCharmProjects/graduation_design/.venv/Lib/site-packages",
"node.js.detected.package.eslint": "true",
"node.js.detected.package.tslint": "true",
"node.js.selected.package.eslint": "(autodetect)",
"node.js.selected.package.tslint": "(autodetect)",
"nodejs_package_manager_path": "npm",
"settings.editor.selected.configurable": "com.jetbrains.python.configuration.PyActiveSdkModuleConfigurable",
"vue.rearranger.settings.migration": "true"
}
}]]></component>
<component name="RecentsManager">
<key name="CopyFile.RECENT_KEYS">
<recent name="F:\Projects\PyCharmProjects\graduation_design\.venv\Lib\site-packages" />
<recent name="F:\Projects\PyCharmProjects\graduation_design\camera\orbbec_camera" />
<recent name="F:\Projects\PyCharmProjects\graduation_design\camera" />
<recent name="F:\Projects\PyCharmProjects\graduation_design\lib" />
<recent name="F:\Projects\PyCharmProjects\graduation_design\integration_testing" />
</key>
<key name="MoveFile.RECENT_KEYS">
<recent name="F:\Projects\PyCharmProjects\graduation_design\camera" />
<recent name="F:\Projects\PyCharmProjects\graduation_design\lib" />
<recent name="F:\Projects\PyCharmProjects\graduation_design\camera\orbbec_camera" />
<recent name="F:\Projects\PyCharmProjects\graduation_design\camera\orbbec_camera\lib" />
<recent name="F:\Projects\PyCharmProjects\graduation_design\yolo" />
</key>
</component>
<component name="SharedIndexes">
<attachedChunks>
<set>
<option value="bundled-python-sdk-5a2391486177-2887949eec09-com.jetbrains.pycharm.pro.sharedIndexes.bundled-PY-233.13763.11" />
</set>
</attachedChunks>
</component>
<component name="SpellCheckerSettings" RuntimeDictionaries="0" Folders="0" CustomDictionaries="0" DefaultDictionary="应用程序级" UseSingleDictionary="true" transferred="true" />
<component name="TaskManager">
<task active="true" id="Default" summary="默认任务">
<changelist id="5de6e434-16d8-46ca-a50f-5cce2ce0877f" name="更改" comment="" />
<created>1708675135052</created>
<option name="number" value="Default" />
<option name="presentableId" value="Default" />
<updated>1708675135052</updated>
<workItem from="1708676369490" duration="2293000" />
<workItem from="1708772527207" duration="4012000" />
<workItem from="1708831408424" duration="3086000" />
<workItem from="1708912830842" duration="533000" />
<workItem from="1708913406749" duration="4000" />
<workItem from="1708913419694" duration="1196000" />
<workItem from="1708915341673" duration="14000" />
<workItem from="1708915370802" duration="660000" />
<workItem from="1708917617291" duration="82000" />
<workItem from="1708917707818" duration="36000" />
<workItem from="1708917750778" duration="29000" />
<workItem from="1708917838563" duration="18000" />
<workItem from="1708917889599" duration="5000" />
<workItem from="1708917964846" duration="246000" />
<workItem from="1708918220728" duration="85000" />
<workItem from="1708918314610" duration="999000" />
<workItem from="1708925007136" duration="10070000" />
<workItem from="1708953749808" duration="5840000" />
</task>
<servers />
</component>
<component name="TypeScriptGeneratedFilesManager">
<option name="version" value="3" />
</component>
<component name="com.intellij.coverage.CoverageDataManagerImpl">
<SUITE FILE_PATH="coverage/graduation_design$yolo.coverage" NAME="yolo 覆盖结果" MODIFIED="1708959490347" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/camera" />
<SUITE FILE_PATH="coverage/graduation_design$image_processing.coverage" NAME="image_processing 覆盖结果" MODIFIED="1708957639566" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/camera" />
<SUITE FILE_PATH="coverage/graduation_design$color_viewer.coverage" NAME="color_viewer 覆盖结果" MODIFIED="1708957257852" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/camera/orbbec_camera" />
</component>
</project>

52
README.md Normal file
View File

@ -0,0 +1,52 @@
## 1. 配置
| **配置项** | **选项** |
|-----------|------------------|
| 🌐 语言 | Python |
| 📚 项目类型 | 通用开发,含计算机视觉和机械控制 |
| 📖 注释风格 | 描述性 + 内联 |
| 🛠️ 代码结构 | 模块化 |
| 🚫 错误处理策略 | 强健 + 上下文 |
| ⚡ 性能优化等级 | 中等 + 可伸缩性关注 |
**注意:** lib中关于双目摄像头的库文件需要依照系统自行编译本项目使用的环境为Windows10
## 2. 设计
### 2.1. 系统概述
- **双目结构光相机系统**: 用于桌面物体的识别和定位。
- **雷达或双目测距系统**: 选择双目测距,因为它和相机系统更协同。
- **机械臂控制系统**: 根据识别和测距结果,控制机械臂移动物体。
### 2.2. 主要模块
1. **图像处理模块**: 使用OpenCV和PyTorch进行图像识别和处理。
2. **双目测距模块**: 计算物体与相机的距离。
3. **机械臂控制模块**: 根据计算结果控制机械臂。
4. **集成与测试模块**: 集成各个模块,并进行测试。
### 2.3. 数据流
1. 相机捕获图像 -> 2. 图像处理识别物体 -> 3. 双目测距计算距离 -> 4. 机械臂根据距离和位置移动物体。
## 3. 项目文件结构
```
plaintext
robotic_arm_project/
|-- camera/
| |-- __init__.py
| |-- image_processing.py
|-- distance_measurement/
| |-- __init__.py
| |-- stereo_vision.py
|-- arm_control/
| |-- __init__.py
| |-- control.py
|-- integration_testing/
| |-- __init__.py
| |-- integration_tests.py
|-- main.py
```

0
camera/__init__.py Normal file
View File

View File

@ -0,0 +1,4 @@
from yolo import process_img
if __name__ == '__main__':
process_img()

View File

View File

@ -0,0 +1,77 @@
# 导入所需模块
from pyorbbecsdk import Pipeline, FrameSet
from pyorbbecsdk import Config
from pyorbbecsdk import OBSensorType, OBFormat
from pyorbbecsdk import OBError
from pyorbbecsdk import VideoStreamProfile
import cv2
from camera.utils import frame_to_bgr_image
# ESC键的键值
ESC_KEY = 27
def main():
# 创建配置对象
config = Config()
# 创建Pipeline对象
pipeline = Pipeline()
try:
# 获取颜色传感器的流配置列表
profile_list = pipeline.get_stream_profile_list(OBSensorType.COLOR_SENSOR)
try:
# 尝试获取指定分辨率、格式和帧率的颜色配置
color_profile: VideoStreamProfile = profile_list.get_video_stream_profile(640, 0, OBFormat.RGB, 30)
except OBError as e:
# 如果出错,打印错误信息并使用默认的颜色配置
print(e)
color_profile = profile_list.get_default_video_stream_profile()
# 启用颜色流
config.enable_stream(color_profile)
except Exception as e:
# 如果出现异常,打印错误信息并返回
print(e)
return
# 启动Pipeline
pipeline.start(config)
while True:
try:
# 等待获取帧集最多等待100毫秒
frames: FrameSet = pipeline.wait_for_frames(100)
if frames is None:
continue
# 获取颜色帧
color_frame = frames.get_color_frame()
if color_frame is None:
continue
# 将帧转换为BGR格式的图像
color_image = frame_to_bgr_image(color_frame)
if color_image is None:
print("failed to convert frame to image")
continue
# 在窗口中显示颜色图像
cv2.imshow("Color Viewer", color_image)
# 等待用户按键每次等待1毫秒
key = cv2.waitKey(1)
# 如果用户按下 'q' 键或 ESC 键,则退出循环
if key == ord('q') or key == ESC_KEY:
break
except KeyboardInterrupt:
# 如果捕获到键盘中断信号,则退出循环
break
# 停止Pipeline
pipeline.stop()
if __name__ == "__main__":
# 当脚本作为主程序运行时调用main函数
main()

126
camera/utils.py Normal file
View File

@ -0,0 +1,126 @@
# ******************************************************************************
# Copyright (c) 2023 Orbbec 3D Technology, Inc
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http:# www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ******************************************************************************
from typing import Union, Any, Optional
from pyorbbecsdk import FormatConvertFilter, VideoFrame
from pyorbbecsdk import OBFormat, OBConvertFormat
import cv2
import numpy as np
def yuyv_to_bgr(frame: np.ndarray, width: int, height: int) -> np.ndarray:
yuyv = frame.reshape((height, width, 2))
bgr_image = cv2.cvtColor(yuyv, cv2.COLOR_YUV2BGR_YUY2)
return bgr_image
def uyvy_to_bgr(frame: np.ndarray, width: int, height: int) -> np.ndarray:
uyvy = frame.reshape((height, width, 2))
bgr_image = cv2.cvtColor(uyvy, cv2.COLOR_YUV2BGR_UYVY)
return bgr_image
def i420_to_bgr(frame: np.ndarray, width: int, height: int) -> np.ndarray:
y = frame[0:height, :]
u = frame[height:height + height // 4].reshape(height // 2, width // 2)
v = frame[height + height // 4:].reshape(height // 2, width // 2)
yuv_image = cv2.merge([y, u, v])
bgr_image = cv2.cvtColor(yuv_image, cv2.COLOR_YUV2BGR_I420)
return bgr_image
def nv21_to_bgr(frame: np.ndarray, width: int, height: int) -> np.ndarray:
y = frame[0:height, :]
uv = frame[height:height + height // 2].reshape(height // 2, width)
yuv_image = cv2.merge([y, uv])
bgr_image = cv2.cvtColor(yuv_image, cv2.COLOR_YUV2BGR_NV21)
return bgr_image
def nv12_to_bgr(frame: np.ndarray, width: int, height: int) -> np.ndarray:
y = frame[0:height, :]
uv = frame[height:height + height // 2].reshape(height // 2, width)
yuv_image = cv2.merge([y, uv])
bgr_image = cv2.cvtColor(yuv_image, cv2.COLOR_YUV2BGR_NV12)
return bgr_image
def determine_convert_format(frame: VideoFrame):
if frame.get_format() == OBFormat.I420:
return OBConvertFormat.I420_TO_RGB888
elif frame.get_format() == OBFormat.MJPG:
return OBConvertFormat.MJPG_TO_RGB888
elif frame.get_format() == OBFormat.YUYV:
return OBConvertFormat.YUYV_TO_RGB888
elif frame.get_format() == OBFormat.NV21:
return OBConvertFormat.NV21_TO_RGB888
elif frame.get_format() == OBFormat.NV12:
return OBConvertFormat.NV12_TO_RGB888
elif frame.get_format() == OBFormat.UYVY:
return OBConvertFormat.UYVY_TO_RGB888
else:
return None
def frame_to_rgb_frame(frame: VideoFrame) -> Union[Optional[VideoFrame], Any]:
if frame.get_format() == OBFormat.RGB:
return frame
convert_format = determine_convert_format(frame)
if convert_format is None:
print("Unsupported format")
return None
print("covert format: {}".format(convert_format))
convert_filter = FormatConvertFilter()
convert_filter.set_format_convert_format(convert_format)
rgb_frame = convert_filter.process(frame)
if rgb_frame is None:
print("Convert {} to RGB failed".format(frame.get_format()))
return rgb_frame
def frame_to_bgr_image(frame: VideoFrame) -> Union[Optional[np.array], Any]:
width = frame.get_width()
height = frame.get_height()
color_format = frame.get_format()
data = np.asanyarray(frame.get_data())
image = np.zeros((height, width, 3), dtype=np.uint8)
if color_format == OBFormat.RGB:
image = np.resize(data, (height, width, 3))
image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)
elif color_format == OBFormat.BGR:
image = np.resize(data, (height, width, 3))
image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
elif color_format == OBFormat.YUYV:
image = np.resize(data, (height, width, 2))
image = cv2.cvtColor(image, cv2.COLOR_YUV2BGR_YUYV)
elif color_format == OBFormat.MJPG:
image = cv2.imdecode(data, cv2.IMREAD_COLOR)
elif color_format == OBFormat.I420:
image = i420_to_bgr(data, width, height)
return image
elif color_format == OBFormat.NV12:
image = nv12_to_bgr(data, width, height)
return image
elif color_format == OBFormat.NV21:
image = nv21_to_bgr(data, width, height)
return image
elif color_format == OBFormat.UYVY:
image = np.resize(data, (height, width, 2))
image = cv2.cvtColor(image, cv2.COLOR_YUV2BGR_UYVY)
else:
print("Unsupported color format: {}".format(color_format))
return None
return image

105
camera/yolo.py Normal file
View File

@ -0,0 +1,105 @@
import cv2
from ultralytics import YOLO
from pyorbbecsdk import Pipeline, FrameSet
from pyorbbecsdk import Config
from pyorbbecsdk import OBSensorType, OBFormat
from pyorbbecsdk import OBError
from pyorbbecsdk import VideoStreamProfile
import numpy as np
from utils import frame_to_bgr_image
def load_model():
# 载入 YOLOv8 模型
model = YOLO('yolov8n.pt')
return model
# 参数说明
# --img: 输入图片的路径
# --is_save_video: 是否保存识别后的视频
def process_img(img=None):
# 暂时性配置
video_path = "output_video.mp4"
# 加载模型
model = load_model()
if img is not None:
# 对图像运行 YOLOv8 推理
results = model(img)
# 在图像上可视化结果
annotated_img = results[0].plot()
# 显示带有标注的图像
cv2.imshow("YOLOv8 推理", annotated_img)
cv2.waitKey(0)
cv2.destroyAllWindows()
else:
# 获取本机摄像头内容
cap = cv2.VideoCapture(0)
# 奥比中光摄像头
# ESC键的键值
ESC_KEY = 27
# 创建配置对象
config = Config()
# 创建Pipeline对象
pipeline = Pipeline()
try:
# 获取颜色传感器的流配置列表
profile_list = pipeline.get_stream_profile_list(OBSensorType.COLOR_SENSOR)
try:
# 尝试获取指定分辨率、格式和帧率的颜色配置
color_profile: VideoStreamProfile = profile_list.get_video_stream_profile(700, 0, OBFormat.RGB, 30)
except OBError as e:
# 如果出错,打印错误信息并使用默认的颜色配置
print(e)
color_profile = profile_list.get_default_video_stream_profile()
# 启用颜色流
config.enable_stream(color_profile)
except Exception as e:
# 如果出现异常,打印错误信息并返回
print(e)
return
# 启动Pipeline
pipeline.start(config)
while True:
try:
# 等待获取帧集最多等待100毫秒
frames: FrameSet = pipeline.wait_for_frames(100)
if frames is None:
continue
# 获取颜色帧
color_frame = frames.get_color_frame()
if color_frame is None:
continue
# 将帧转换为BGR格式的图像
color_image = frame_to_bgr_image(color_frame)
model.predict(source=color_image, show=True)
if color_image is None:
print("failed to convert frame to image")
continue
key = cv2.waitKey(1)
if key == ord('q') or key == ESC_KEY:
break
except KeyboardInterrupt:
# 如果捕获到键盘中断信号,则退出循环
break
# 释放视频捕获对象并关闭显示窗口
cap.release()
cv2.destroyAllWindows()
# 停止Pipeline
pipeline.stop()
if __name__ == '__main__':
process_img()

BIN
camera/yolov8n.pt Normal file

Binary file not shown.

View File

View File

@ -0,0 +1,22 @@
import numpy as np
import cv2
# 双目测距模块
# 双目相机校准
def stereo_calibration(left_camera_params, right_camera_params):
# TODO: 实现双目相机的校准
pass
# 计算视差
def compute_disparity(left_image, right_image):
# TODO: 计算左右图像的视差
pass
# 计算距离
def measure_distance(disparity_map, calibration_data):
# TODO: 使用视差图和校准数据计算距离
pass

View File

View File

12
main.py Normal file
View File

@ -0,0 +1,12 @@
import cv2
import torch
def process_image(image):
# todo:使用OpenCV和Torch处理图像识别物体
pass
def get_object_coordinates(image):
# todo返回识别物体的坐标
pass

View File

26
robot_control/control.py Normal file
View File

@ -0,0 +1,26 @@
# TODO: 导入机械臂控制相关的库根据实际使用的SDK
# 机械臂控制
def initialize_arm_control():
# TODO: 初始化机械臂控制
pass
def move_arm_to_position(x, y, z):
# TODO: 移动机械臂到指定位置
pass
def grab_object():
# TODO: 控制机械臂抓取物体
pass
def release_object():
# TODO: 控制机械臂释放物体
pass
def move_object_to_location(object_coords, target_coords):
# TODO: 将物体从当前位置移动到目标位置
pass