rdkartono/BirdDeterrentSystem
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Patch 17/03/2025

Browse Source
This commit is contained in:
rdkartono
2025-03-17 13:35:05 +07:00
parent 123a89eec9
commit a1b6ec54ab
13 changed files with 640 additions and 211 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
Config/config.properties
View File

@@ -4,6 +4,7 @@ Camera_port = 80
Camera_user = root
Camera_password = password
Camera_Rtsp_path = /axis-media/media.amp
#Camera_Rtsp_path = /video1
AudioFile01 = elangWav.wav
AudioFile02 = gunshotsWav.wav
AudioFile03 = pinkNoiseWav.wav

3
config.properties
View File

@@ -2,10 +2,11 @@
AudioFile01=elangWav.wav
AudioFile02=gunshotsWav.wav
AudioFile03=pinkNoiseWav.wav
AudioFile04=
AudioFile04=null
AudioFile05=null
AudioVolumeOutput=100
Camera_Rtsp_path=/axis-media/media.amp
#Camera_Rtsp_path=/video1
Camera_ip=172.17.195.51
Camera_password=password
Camera_port=80

BIN
onnx/yolo11n.onnx → onnx/yolov8n.onnx
View File

Binary file not shown.

3
src/main/java/Audio/MultiUSBAudioPlayer.java
View File

@@ -72,6 +72,7 @@ public class MultiUSBAudioPlayer {
List<Integer> result = new ArrayList<>();
for(BassDeviceInfoWithId dev : playbackdevices){
if (dev.info.name.toLowerCase().contains(name.toLowerCase())){
Logger.info("Found Device id={}, name={}", dev.id, dev.info.name);
result.add(dev.id);
}
}
@@ -98,7 +99,7 @@ public class MultiUSBAudioPlayer {
validdevice.add(i);
inited = true;
} else {
Logger.error("Failed to initialize device {}", i);
Logger.error("Failed to initialize device {}, errorcode={}", i, bass.BASS_ErrorGetCode());
}
}
deviceid = validdevice.stream().mapToInt(i->i).toArray();

54
src/main/java/Camera/GrabbingTask.java
View File

@@ -5,14 +5,14 @@ import java.util.concurrent.atomic.AtomicBoolean;
import java.util.function.Consumer;
import Other.SomeCodes;
import id.co.gtc.Main;
import lombok.Getter;
import lombok.Setter;
import org.bytedeco.javacv.Frame;
import org.bytedeco.javacv.FrameGrabber;
import org.bytedeco.javacv.OpenCVFrameConverter;
import org.opencv.core.Mat;
import org.opencv.core.Size;
import org.opencv.imgproc.Imgproc;
import org.opencv.core.*;
import org.tinylog.Logger;
public class GrabbingTask implements Runnable {
@@ -39,8 +39,8 @@ public class GrabbingTask implements Runnable {
// for FPS calculation
private int intendedFps = 10;
private final boolean useYolo;
private final YoloDetector yolo;
private final YoloDetector_opencv yolo;
@SuppressWarnings("SameParameterValue")
private void updateMessage(String message) {
@@ -90,12 +90,13 @@ public class GrabbingTask implements Runnable {
public GrabbingTask(AtomicBoolean isGrabbing, FrameGrabber grabber, int fps, boolean useYolo) {
public GrabbingTask(AtomicBoolean isGrabbing, FrameGrabber grabber, int fps) {
this.isGrabbing = isGrabbing;
this.grabber = grabber;
if (fps>0) intendedFps = fps;
this.useYolo = useYolo;
if (useYolo) yolo = new YoloDetector(); else yolo = null;
if (Main.use_Yolo) {
yolo = new YoloDetector_opencv();
} else yolo = null;
}
@@ -104,28 +105,27 @@ public class GrabbingTask implements Runnable {
}
private void processFrame(Frame fr){
@SuppressWarnings({"CallToPrintStackTrace", "CatchMayIgnoreException"})
private Mat processFrame(Frame fr){
if (fr!=null && fr.image!=null && fr.imageWidth>0 && fr.imageHeight>0){
try(var converter = new OpenCVFrameConverter.ToOrgOpenCvCoreMat()){
org.opencv.core.Mat mat = converter.convert(fr);
Mat mat = converter.convert(fr);
if (mat.cols()>0 && mat.rows()>0){
updateHQFrame(fr);
updateHQBase64(SomeCodes.MatToBase64(mat));
Size lowsize = new Size(lowquality_width, lowquality_height);
Mat resized = new Mat();
Imgproc.resize(mat, resized, lowsize);
mat.release();
if (resized.cols()>0 && resized.rows()>0){
Frame resizedFrame = SomeCodes.CoreMatConverter.convert(resized);
Frame resizedFrame = converter.convert(resized);
updateLQFrame(resizedFrame);
updateLQBase64(SomeCodes.MatToBase64(resized));
if (useYolo){
DetectYolo(resized);
}
return resized;
} else Logger.error("processFrame resized size is 0");
resized.release();
} else Logger.error("processFrame Mat size is 0");
mat.release();
} catch (Exception e){
if (SomeCodes.ValidString(e.getMessage())) {
if (!e.getMessage().startsWith("unknown exception")) {
@@ -136,10 +136,12 @@ public class GrabbingTask implements Runnable {
}
} else {
Logger.warn("processFrame have null frame");
updateMessage("processFrame have null frame");
}
// else {
// Logger.warn("processFrame have null frame");
// updateMessage("processFrame have null frame");
// }
return null;
}
private void DetectYolo(Mat mat){
@@ -175,26 +177,31 @@ public class GrabbingTask implements Runnable {
}
}
@SuppressWarnings("CallToPrintStackTrace")
@Override
public void run() {
isGrabbing.set(true);
Logger.info("Grabbing Task started");
double fps = grabber.getFrameRate();
int skippedframes = (int)(fps / intendedFps);
//Logger.info("Grabber framerate = {}, intendedFps = {}, Skipping frames = {}", fps, intendedFps, skippedframes);
Logger.info("Grabber framerate = {}, intendedFps = {}, Skipping frames = {}", fps, intendedFps, skippedframes);
int framecount = 0;
flush_grabber();
long starttick = System.currentTimeMillis();
while (isGrabbing.get()) {
try{
Thread.yield();
Frame frame = grabber.grab();
if (framecount<Integer.MAX_VALUE) framecount++; else framecount = 0;
//processFrame(frame);
Mat resized = processFrame(frame);
// no need to skip frames, just process all frames
if (framecount % skippedframes == 0) {
processFrame(frame);
if (Main.use_Yolo){
DetectYolo(resized);
}
}
resized.release();
} catch (Exception e){
Logger.error("Error grabbing frame: "+e.getMessage());
e.printStackTrace();
@@ -205,7 +212,8 @@ public class GrabbingTask implements Runnable {
starttick = System.currentTimeMillis();
if (elapsed>0) {
int xx = (int) (1000 / elapsed);
if (xx<fps) CaptureFPS = xx;
//if (xx<fps) CaptureFPS = xx;
CaptureFPS = xx;
}
//Logger.info("Elapsed time = {} ms, captureFPS = {}", elapsed, CaptureFPS);
}

45
src/main/java/Camera/PanTiltController.java
View File

@@ -70,9 +70,10 @@ public class PanTiltController {
command[0] = (byte) 0xFF; // add synchronization byte
if (isOpen()) {
Main.Max485Direction(true);
serialPort.writeBytes(command, command.length);
int written = serialPort.writeBytes(command, command.length);
Main.Max485Direction(false);
Main.Blink_LedPanTilt();
Logger.info("Stop movement written {}", written);
} else Logger.warn("Stop Movement failed, serial port not open");
}
@@ -89,9 +90,10 @@ public class PanTiltController {
command[0] = (byte) 0xFF; // add synchronization byte
if (isOpen()) {
Main.Max485Direction(true);
serialPort.writeBytes(command, command.length);
int written = serialPort.writeBytes(command, command.length);
Main.Max485Direction(false);
Main.Blink_LedPanTilt();
Logger.info("Pan Left written {} bytes", written);
} else Logger.warn("Pan Left failed, serial port not open");
}
@@ -108,9 +110,10 @@ public class PanTiltController {
command[0] = (byte) 0xFF; // add synchronization byte
if (isOpen()) {
Main.Max485Direction(true);
serialPort.writeBytes(command, command.length);
int written = serialPort.writeBytes(command, command.length);
Main.Max485Direction(false);
Main.Blink_LedPanTilt();
Logger.info("Pan Right written {} bytes", written);
} else Logger.warn("Pan Right failed, serial port not open");
}
@@ -122,14 +125,15 @@ public class PanTiltController {
if (speed<0) speed = 0;
if (speed>0x3F) speed = 0x3F;
byte[] command = new byte[]{0, cameraid, 0, 8, speed, 0, 0};
byte[] command = new byte[]{0, cameraid, 0, 8, 0, speed, 0};
command[6] = Checksum(command); // add checksum
command[0] = (byte) 0xFF; // add synchronization byte
if (isOpen()) {
Main.Max485Direction(true);
serialPort.writeBytes(command, command.length);
int written = serialPort.writeBytes(command, command.length);
Main.Max485Direction(false);
Main.Blink_LedPanTilt();
Logger.info("Tilt Up written {} bytes", written);
} else Logger.warn("Tilt Up failed, serial port not open");
}
@@ -141,17 +145,44 @@ public class PanTiltController {
if (speed<0) speed = 0;
if (speed>0x3F) speed = 0x3F;
byte[] command = new byte[]{0, cameraid, 0, 16, speed, 0, 0};
byte[] command = new byte[]{0, cameraid, 0, 16, 0, speed, 0};
command[6] = Checksum(command); // add checksum
command[0] = (byte) 0xFF; // add synchronization byte
if (isOpen()) {
Main.Max485Direction(true);
serialPort.writeBytes(command, command.length);
int written = serialPort.writeBytes(command, command.length);
Main.Max485Direction(false);
Main.Blink_LedPanTilt();
Logger.info("Tilt Down written {} bytes", written);
} else Logger.warn("Tilt Down failed, serial port not open");
}
public void DisableMovementTest(){
byte[] command = new byte[]{(byte)0xFF, cameraid, 0, 7, 0, (byte)0x54, (byte)0x5c};
//command[6] = Checksum(command);
//command[0] = (byte) 0xFF;
if (isOpen()) {
Main.Max485Direction(true);
int written = serialPort.writeBytes(command, command.length);
Main.Max485Direction(false);
Main.Blink_LedPanTilt();
Logger.info("DisableMovement test written {} bytes", written);
} else Logger.warn("DisableMovement failed, serial port not open");
}
public void EnableMovementTest(){
byte[] command = new byte[]{(byte)0xFF, cameraid, 0, 3, 0, (byte)0x54, (byte) 0x58};
//command[6] = Checksum(command);
//command[0] = (byte) 0xFF;
if (isOpen()) {
Main.Max485Direction(true);
int written = serialPort.writeBytes(command, command.length);
Main.Max485Direction(false);
Main.Blink_LedPanTilt();
Logger.info("EnableMovement test written {} bytes", written);
} else Logger.warn("EnableMovement failed, serial port not open");
}
/**
* Go to Preset
* @param preset preset number

7
src/main/java/Camera/RtspGrabber.java
View File

@@ -3,7 +3,6 @@ import lombok.Getter;
import org.bytedeco.ffmpeg.global.avutil;
import org.bytedeco.javacv.FFmpegFrameGrabber;
import org.bytedeco.javacv.Frame;
import org.bytedeco.opencv.global.opencv_core;
import org.jetbrains.annotations.NotNull;
import org.tinylog.Logger;
@@ -26,11 +25,9 @@ public class RtspGrabber {
private @Getter int HQHeight = 0;
private @Getter int LQWidth = 0;
private @Getter int LQHeight = 0;
private final boolean useYolo;
private GrabbingTask grabbingTask;
public RtspGrabber(String ip, String path, boolean useYolo) {
this.useYolo = useYolo;
public RtspGrabber(String ip, String path) {
this.rtspUrl = "rtsp://" + ip + path;
Logger.info("RtspGrabber created with url: " + rtspUrl);
@@ -142,7 +139,7 @@ public class RtspGrabber {
@NotNull
private GrabbingTask getGrabbingTask() {
GrabbingTask tt = new GrabbingTask(isGrabbing, grabber,10, useYolo);
GrabbingTask tt = new GrabbingTask(isGrabbing, grabber,10);
tt.setOnMessageUpdate(Logger::info);
tt.setOnHQFrameUpdate(value -> {
if (value!=null){

169
src/main/java/Camera/RtspGrabber_opencv.java Normal file
View File

@@ -0,0 +1,169 @@
package Camera;
import lombok.Getter;
import org.bytedeco.ffmpeg.global.avutil;
import org.bytedeco.javacv.FFmpegFrameGrabber;
import org.bytedeco.javacv.Frame;
import org.jetbrains.annotations.NotNull;
import org.tinylog.Logger;
import java.util.concurrent.atomic.AtomicBoolean;
import static Other.SomeCodes.gson;
@SuppressWarnings("unused")
public class RtspGrabber_opencv {
private final String rtspUrl;
private FFmpegFrameGrabber grabber;
private final AtomicBoolean isGrabbing = new AtomicBoolean(false);
private Frame lastHQFrame = null;
private Frame lastLQFrame = null;
private Frame lastYoloFrame = null;
private String lastHQBase64 = null;
private String lastLQBase64 = null;
private String lastYoloBase64 = null;
private @Getter int HQWidth = 0;
private @Getter int HQHeight = 0;
private @Getter int LQWidth = 0;
private @Getter int LQHeight = 0;
private GrabbingTask grabbingTask;
public RtspGrabber_opencv(String ip, String path) {
this.rtspUrl = "rtsp://" + ip + path;
Logger.info("RtspGrabber created with url: " + rtspUrl);
}
private synchronized void setLastHQFrame(Frame frame){
lastHQFrame = frame;
}
public synchronized Frame getLastHQFrame(){
return lastHQFrame;
}
private synchronized void setLastLQFrame(Frame frame){
lastLQFrame = frame;
}
public synchronized Frame getLastLQFrame(){
return lastLQFrame;
}
private synchronized void setLastHQBase64(String base64){
lastHQBase64 = base64;
}
public synchronized String getLastHQBase64(){
return lastHQBase64;
}
private synchronized void setLastLQBase64(String base64){
lastLQBase64 = base64;
}
public synchronized String getLastLQBase64(){
return lastLQBase64;
}
private synchronized void setLastYoloBase64(String base64){
lastYoloBase64 = base64;
}
public synchronized String getLastYoloBase64(){
return lastYoloBase64;
}
private synchronized void setLastYoloFrame(Frame frame){
lastYoloFrame = frame;
}
public synchronized Frame getLastYoloFrame(){
return lastYoloFrame;
}
/**
* Start grabbing frames from rtsp
* @param useTcp Use tcp instead of udp
*/
public void Start(boolean useTcp, final int width, final int height){
try{
grabber = FFmpegFrameGrabber.createDefault(rtspUrl);
if (useTcp) grabber.setOption("rtsp_transport", "tcp");
grabber.setPixelFormat(avutil.AV_PIX_FMT_BGR24);
grabber.start();
avutil.av_log_set_level(avutil.AV_LOG_ERROR);
Logger.info("Grabber started");
GrabbingTask tt = getGrabbingTask();
grabbingTask = tt;
new Thread(tt).start();
} catch (Exception e){
Logger.error("Error starting grabber: " + e.getMessage());
}
}
/**
* Stop grabbing frames
*/
public void Stop(){
isGrabbing.set(false);
if (grabbingTask!=null){
grabbingTask.Stop();
}
grabbingTask = null;
if (grabber!=null) {
try{
grabber.stop();
grabber.releaseUnsafe();
Logger.info("Grabber stopped");
} catch (Exception e){
Logger.error("Error stopping grabber: " + e.getMessage());
}
}
grabber = null;
}
public String LQStreamingStatus(){
return gson.toJson(new String[]{String.valueOf(LQWidth), String.valueOf(LQHeight) , String.valueOf(grabbingTask!=null ? grabbingTask.getCaptureFPS():"0")});
}
public String HQStreamingStatus(){
return gson.toJson(new String[]{String.valueOf(HQWidth), String.valueOf(HQHeight) , String.valueOf(grabbingTask!=null ? grabbingTask.getCaptureFPS(): "0")});
}
@NotNull
private GrabbingTask getGrabbingTask() {
GrabbingTask tt = new GrabbingTask(isGrabbing, grabber,5);
tt.setOnMessageUpdate(Logger::info);
tt.setOnHQFrameUpdate(value -> {
if (value!=null){
if (value.imageWidth>0 && value.imageHeight>0){
setLastHQFrame(value);
HQWidth = value.imageWidth;
HQHeight = value.imageHeight;
}
}
});
tt.setOnLQFrameUpdate(value -> {
if (value!=null){
if (value.imageWidth>0 && value.imageHeight>0){
setLastLQFrame(value);
LQWidth = value.imageWidth;
LQHeight = value.imageHeight;
}
}
});
tt.setOnHQBase64Update(this::setLastHQBase64);
tt.setOnLQBase64Update(this::setLastLQBase64);
tt.setOnYoloUpdate(this::setLastYoloFrame);
tt.setOnYoloBase64Update(this::setLastYoloBase64);
return tt;
}
}

219
src/main/java/Camera/YoloDetector.java
View File

@@ -1,11 +1,17 @@
package Camera;
import Other.SomeCodes;
import id.co.gtc.Main;
import lombok.Getter;
import org.opencv.core.*;
import org.opencv.dnn.Dnn;
import org.opencv.dnn.Net;
import org.bytedeco.javacpp.DoublePointer;
import org.bytedeco.javacpp.FloatPointer;
import org.bytedeco.javacpp.IntPointer;
import org.bytedeco.opencv.global.opencv_core;
import org.bytedeco.opencv.global.opencv_dnn;
import org.bytedeco.opencv.global.opencv_imgproc;
import org.bytedeco.opencv.opencv_core.*;
import org.bytedeco.opencv.opencv_dnn.Net;
import org.opencv.imgproc.Imgproc;
import org.tinylog.Logger;
@@ -22,47 +28,46 @@ public class YoloDetector {
private @Getter boolean NetLoaded;
private List<String> classes ;
// minimum confidence for detections
private final float confidence_threshold = 0.4f;
// Non-maximum suppression threshold
private final float nms_threshold = 0.4f;
public YoloDetector() {
NetLoaded = false;
net = null;
classes = null;
String onnxfile = SomeCodes.ExtractResource("/yolo11n.onnx", SomeCodes.currentDirectory);
String namesfile = SomeCodes.ExtractResource("/coco.names", SomeCodes.currentDirectory);
if (SomeCodes.ValidFile(onnxfile)){
if (SomeCodes.ValidFile(namesfile)){
net = Dnn.readNetFromONNX(onnxfile);
net.setPreferableBackend(Dnn.DNN_BACKEND_CUDA);
net.setPreferableTarget(Dnn.DNN_TARGET_CUDA);
//net.setPreferableBackend(Dnn.DNN_BACKEND_OPENCV);
//net.setPreferableBackend(Dnn.DNN_BACKEND_INFERENCE_ENGINE);
//net.setPreferableTarget(Dnn.DNN_TARGET_CPU);
//net.setPreferableTarget(Dnn.DNN_TARGET_NPU);
String netfile = SomeCodes.ExtractResource("/yolov8n.onnx", SomeCodes.currentDirectory);
if (SomeCodes.ValidFile(netfile)){
net = opencv_dnn.readNetFromONNX(netfile);
if (Main.haveCuda){
net.setPreferableBackend(opencv_dnn.DNN_BACKEND_CUDA);
net.setPreferableTarget(opencv_dnn.DNN_TARGET_CUDA_FP16);
Logger.info("Net loaded with CUDA");
} else {
net.setPreferableBackend(opencv_dnn.DNN_BACKEND_OPENCV);
net.setPreferableTarget(opencv_dnn.DNN_TARGET_CPU);
Logger.info("Net loaded with CPU");
}
} else Logger.error("net file not found");
classes = new ArrayList<>();
if (namesfile!=null){
try(BufferedReader br = new BufferedReader(new FileReader(namesfile))) {
String line;
while ((line = br.readLine()) != null) {
classes.add(line);
}
} catch (Exception e){
Logger.error("reading names file, Exception : ",e.getMessage());
if (net!=null){
classes = new ArrayList<>();
if (namesfile!=null){
try(BufferedReader br = new BufferedReader(new FileReader(namesfile))) {
String line;
while ((line = br.readLine()) != null) {
classes.add(line);
}
} catch (Exception e){
Logger.error("reading names file, Exception : ",e.getMessage());
}
}
if (!net.empty()){
if (!classes.isEmpty()){
NetLoaded = true;
} else Logger.error("names file is empty");
} else Logger.error("net is empty");
} else Logger.error("Net is not loaded");
if (!net.empty()){
if (!classes.isEmpty()){
NetLoaded = true;
} else Logger.error("names file is empty");
} else Logger.error("net is empty");
} else Logger.error("names file not found");
} else Logger.error("onnx file not found");
}
// Source : https://blog.csdn.net/taoli188/article/details/134720614
@@ -71,12 +76,13 @@ public class YoloDetector {
if (NetLoaded){
Size inputSize = new Size(640, 640);
Scalar mean = new Scalar(0, 0, 0, 0);
float scaleFactor = 1.0f/255.0f;
double scaleFactor = 1.0f/255.0f;
boolean swapRB = true;
boolean crop = false;
Mat blob = Dnn.blobFromImage(frame, scaleFactor, inputSize, mean , swapRB, crop, CV_32F);
//Mat blob = Dnn.blobFromImage(frame, scaleFactor, inputSize);
//Mat blob = opencv_dnn.blobFromImage(frame, scaleFactor, inputSize, mean , swapRB, crop, CV_32F);
Mat blob = new Mat();
opencv_dnn.blobFromImage(frame, blob, scaleFactor, inputSize, mean, swapRB, crop, CV_32F);
net.setInput(blob);
Mat predict = net.forward();
@@ -94,8 +100,10 @@ public class YoloDetector {
}
@SuppressWarnings("CallToPrintStackTrace")
public Mat Process(Mat frame, Mat predict, Mat mask){
try{
double width = frame.cols() / 640.0;
double height = frame.rows() / 640.0;
Rect2d[] rect2d = new Rect2d[mask.cols()];
@@ -103,28 +111,48 @@ public class YoloDetector {
int[] classid = new int[mask.cols()];
for(int i=0;i<mask.cols();i++){
double[] x = mask.col(i).get(0, 0);
double[] y = mask.col(i).get(1, 0);
double[] w = mask.col(i).get(2, 0);
double[] h = mask.col(i).get(3, 0);
double[] x = SomeCodes.getColumnValues(mask,i, new int[]{0});
double[] y = SomeCodes.getColumnValues(mask,i, new int[]{1});
double[] w = SomeCodes.getColumnValues(mask,i, new int[]{2});
double[] h = SomeCodes.getColumnValues(mask,i, new int[]{3});
rect2d[i] = new Rect2d((x[0]-w[0]/2)*width, (y[0]-h[0]/2)*height, w[0]*width, h[0]*height);
Mat score = mask.col(i).submat(4, predict.size(1)-1,0,1) ;
Core.MinMaxLocResult mm = Core.minMaxLoc(score);
scoref[i] = (float)mm.maxVal;
classid[i] = (int)mm.maxLoc.y;
//Mat score = mask.col(i).submat(4, predict.size(1)-1,0,1) ;
Mat score = SomeCodes.getSubmat(mask,i,4,predict.size(1)-1);
// Core.MinMaxLocResult mm = Core.minMaxLoc(score);
// scoref[i] = (float)mm.maxVal;
// classid[i] = (int)mm.maxLoc.y;
DoublePointer minVal = new DoublePointer(1);
DoublePointer maxVal = new DoublePointer(1);
Point minLoc = new Point();
Point maxLoc = new Point();
opencv_core.minMaxLoc(score, minVal, maxVal, minLoc, maxLoc, null);
scoref[i] = (float)maxVal.get();
classid[i] = (int)maxLoc.y();
}
MatOfRect2d bboxes = new MatOfRect2d(rect2d);
MatOfFloat scores = new MatOfFloat(scoref);
MatOfInt indices = new MatOfInt();
Dnn.NMSBoxes(bboxes, scores, confidence_threshold, nms_threshold, indices);
//MatOfRect2d bboxes = new MatOfRect2d(rect2d);
Rect2dVector bboxes = new Rect2dVector(rect2d);
//MatOfFloat scores = new MatOfFloat(scoref);
FloatPointer scores = new FloatPointer(scoref);
//MatOfInt indices = new MatOfInt();
//IntVector indices = new IntVector();
IntPointer indices = new IntPointer();
// minimum confidence for detections
float confidence_threshold = 0.4f;
// Non-maximum suppression threshold
float nms_threshold = 0.4f;
if (indices!=null && !indices.empty()){
int[] idx = indices.toArray();
opencv_dnn.NMSBoxes(bboxes, scores, confidence_threshold, nms_threshold, indices);
if (indices.limit()>0){
int[] idx = new int[(int)indices.limit()];
indices.get(idx);
for(int ii : idx){
Rect rr = new Rect(rect2d[ii].tl(), rect2d[ii].size());
Imgproc.rectangle(frame, rr, new Scalar(0, 255, 0), 2);
Imgproc.putText(frame, classes.get(classid[ii])+" "+String.format("%.2f", scoref[ii]), rr.tl(), Imgproc.FONT_HERSHEY_SIMPLEX, 1.0, new Scalar(0, 0, 255));
Rect rr = new Rect(SomeCodes.toPoint(rect2d[ii].tl()), SomeCodes.toSize(rect2d[ii].size()));
opencv_imgproc.rectangle(frame,rr, Scalar.GREEN);
opencv_imgproc.putText(frame, classes.get(classid[ii])+" "+String.format("%.2f", scoref[ii]), rr.tl(), Imgproc.FONT_HERSHEY_SIMPLEX, 1.0, Scalar.RED);
}
}
@@ -138,87 +166,4 @@ public class YoloDetector {
return frame;
}
public Mat processDetection(Mat frame, Mat output){
int rows = output.rows();
int cols = output.cols();
List<Rect> boxes = new ArrayList<>();
List<Float> confidences = new ArrayList<>();
List<Integer> classIds = new ArrayList<>();
for (int i = 0; i < rows; i++){
Mat row = output.row(i);
Mat scores = row.colRange(5, cols);
Core.MinMaxLocResult mm = Core.minMaxLoc(scores);
float confidence = (float)mm.maxVal;
Point classIdPoint = mm.maxLoc;
if (confidence>confidence_threshold){
int centerX = (int)(row.get(0, 0)[0] * frame.cols());
int centerY = (int)(row.get(0, 1)[0] * frame.rows());
int width = (int)(row.get(0, 2)[0] * frame.cols());
int height = (int)(row.get(0, 3)[0] * frame.rows());
int left = centerX - width / 2;
int top = centerY - height / 2;
boxes.add(new Rect(left, top, width, height));
confidences.add(confidence);
classIds.add((int)classIdPoint.x);
}
}
MatOfRect2d rects = new MatOfRect2d(ToRect2D(boxes));
MatOfFloat conf = new MatOfFloat(ToFloatArray(confidences));
MatOfInt ids = new MatOfInt();
Dnn.NMSBoxes(rects, conf, confidence_threshold, nms_threshold, ids);
int[] indices = ids.toArray();
if (indices.length>0){
for (int idx : indices){
Rect box = boxes.get(idx);
int classId = classIds.get(idx);
float confidence = confidences.get(idx);
String label = classes.get(classId);
Logger.info("Detect ["+idx+"] ClassID: "+classId+" Label: "+label+" Confidence: "+confidence);
Imgproc.rectangle(frame, box, new Scalar(0, 255, 0), 2);
String text = label + " " + String.format("%.2f", confidence);
int[] baseLine = new int[1];
Size labelSize = Imgproc.getTextSize(text, Imgproc.FONT_HERSHEY_SIMPLEX, 0.5, 1, baseLine);
Imgproc.rectangle(frame, new Point(box.x, box.y - labelSize.height),
new Point(box.x + labelSize.width, box.y + baseLine[0]),
new Scalar(255, 255, 255), Imgproc.FILLED);
Imgproc.putText(frame, text, new Point(box.x, box.y),
Imgproc.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar(0, 0, 0));
}
}
return frame;
}
private Rect2d[] ToRect2D(List<Rect> list){
if (list!=null && list.size()>0){
Rect2d array[] = new Rect2d[list.size()];
for (int i = 0; i < list.size(); i++){
Rect r = list.get(i);
array[i] = new Rect2d(r.x, r.y, r.width, r.height);
}
return array;
}
return new Rect2d[0];
}
private float[] ToFloatArray(List<Float> list){
if (list!=null && list.size()>0){
float array[] = new float[list.size()];
for (int i = 0; i < list.size(); i++){
array[i] = list.get(i);
}
return array;
}
return new float[0];
}
}

146
src/main/java/Camera/YoloDetector_opencv.java Normal file
View File

@@ -0,0 +1,146 @@
package Camera;
import Other.SomeCodes;
import id.co.gtc.Main;
import lombok.Getter;
import org.opencv.core.*;
import org.opencv.dnn.Dnn;
import org.opencv.dnn.Net;
import org.opencv.imgproc.Imgproc;
import org.tinylog.Logger;
import java.io.BufferedReader;
import java.io.FileReader;
import java.util.ArrayList;
import java.util.List;
import static org.bytedeco.opencv.global.opencv_core.CV_32F;
public class YoloDetector_opencv {
private Net net;
private @Getter boolean NetLoaded;
private List<String> classes ;
public YoloDetector_opencv() {
NetLoaded = false;
net = null;
classes = null;
String namesfile = SomeCodes.ExtractResource("/coco.names", SomeCodes.currentDirectory);
String netfile = SomeCodes.ExtractResource("/yolov8n.onnx", SomeCodes.currentDirectory);
if (SomeCodes.ValidFile(netfile)){
net = Dnn.readNetFromONNX(netfile);
if (Main.haveCuda){
net.setPreferableBackend(Dnn.DNN_BACKEND_CUDA);
net.setPreferableTarget(Dnn.DNN_TARGET_CUDA_FP16);
Logger.info("Net loaded with CUDA");
} else {
net.setPreferableBackend(Dnn.DNN_BACKEND_OPENCV);
net.setPreferableTarget(Dnn.DNN_TARGET_CPU);
Logger.info("Net loaded with CPU");
}
} else Logger.error("net file not found");
if (net!=null){
classes = new ArrayList<>();
if (namesfile!=null){
try(BufferedReader br = new BufferedReader(new FileReader(namesfile))) {
String line;
while ((line = br.readLine()) != null) {
classes.add(line);
}
} catch (Exception e){
Logger.error("reading names file, Exception : ",e.getMessage());
}
}
if (!net.empty()){
if (!classes.isEmpty()){
NetLoaded = true;
} else Logger.error("names file is empty");
} else Logger.error("net is empty");
} else Logger.error("Net is not loaded");
}
// Source : https://blog.csdn.net/taoli188/article/details/134720614
public Mat[] Detect(Mat frame){
try{
if (NetLoaded){
Size inputSize = new Size(640, 640);
Scalar mean = new Scalar(0, 0, 0, 0);
float scaleFactor = 1.0f/255.0f;
boolean swapRB = true;
boolean crop = false;
Mat blob = Dnn.blobFromImage(frame, scaleFactor, inputSize, mean , swapRB, crop, CV_32F);
//Mat blob = Dnn.blobFromImage(frame, scaleFactor, inputSize);
net.setInput(blob);
Mat predict = net.forward();
Mat mask = predict.reshape(0,1).reshape(0, predict.size(1));
return new Mat[]{predict,mask};
} else {
Logger.error("Net not loaded");
return new Mat[0];
}
} catch (Exception e){
Logger.error("Error detecting: "+e.getMessage());
return new Mat[0];
}
}
@SuppressWarnings("CallToPrintStackTrace")
public Mat Process(Mat frame, Mat predict, Mat mask){
try{
double width = frame.cols() / 640.0;
double height = frame.rows() / 640.0;
Rect2d[] rect2d = new Rect2d[mask.cols()];
float[] scoref = new float[mask.cols()];
int[] classid = new int[mask.cols()];
for(int i=0;i<mask.cols();i++){
double[] x = mask.col(i).get(0, 0);
double[] y = mask.col(i).get(1, 0);
double[] w = mask.col(i).get(2, 0);
double[] h = mask.col(i).get(3, 0);
rect2d[i] = new Rect2d((x[0]-w[0]/2)*width, (y[0]-h[0]/2)*height, w[0]*width, h[0]*height);
Mat score = mask.col(i).submat(4, predict.size(1)-1,0,1) ;
Core.MinMaxLocResult mm = Core.minMaxLoc(score);
scoref[i] = (float)mm.maxVal;
classid[i] = (int)mm.maxLoc.y;
}
MatOfRect2d bboxes = new MatOfRect2d(rect2d);
MatOfFloat scores = new MatOfFloat(scoref);
MatOfInt indices = new MatOfInt();
// minimum confidence for detections
float confidence_threshold = 0.4f;
// Non-maximum suppression threshold
float nms_threshold = 0.4f;
Dnn.NMSBoxes(bboxes, scores, confidence_threshold, nms_threshold, indices);
if (!indices.empty()){
int[] idx = indices.toArray();
for(int ii : idx){
Rect rr = new Rect(rect2d[ii].tl(), rect2d[ii].size());
Imgproc.rectangle(frame, rr, new Scalar(0, 255, 0), 2);
Imgproc.putText(frame, classes.get(classid[ii])+" "+String.format("%.2f", scoref[ii]), rr.tl(), Imgproc.FONT_HERSHEY_SIMPLEX, 1.0, new Scalar(0, 0, 255));
}
}
} catch (Exception e){
Logger.error("Error processing detection: "+e.getMessage());
e.printStackTrace();
}
return frame;
}
}

115
src/main/java/Other/SomeCodes.java
View File

@@ -2,13 +2,14 @@ package Other;
import com.google.gson.Gson;
import com.sun.jna.Platform;
import org.bytedeco.javacpp.Loader;
import org.bytedeco.javacpp.BytePointer;
import org.bytedeco.javacpp.indexer.DoubleIndexer;
import org.bytedeco.javacv.Frame;
import org.bytedeco.javacv.OpenCVFrameConverter;
import org.bytedeco.opencv.global.opencv_core;
import org.bytedeco.opencv.global.opencv_imgcodecs;
import org.bytedeco.opencv.global.opencv_imgproc;
import org.bytedeco.opencv.opencv_core.*;
import org.bytedeco.opencv.opencv_java;
import org.jetbrains.annotations.NotNull;
import org.opencv.core.MatOfByte;
import org.opencv.imgcodecs.Imgcodecs;
@@ -24,11 +25,7 @@ public class SomeCodes {
static{
//Loader.load(opencv_java.class);
if (Platform.isLinux()){
File ff = new File("/usr/local/lib/libopencv_java4100.so");
if (ff.isFile()){
Logger.info("Loading opencv_java 4.10.0 with cuda from /usr/local/lib");
System.load(ff.getAbsolutePath());
} else Logger.info("library opencv_java with cuda not found, loading standard opencv");
LoadLinuxLibrary("/usr/local/lib/libopencv_java4100.so");
} else Logger.info("Platform is not Linux, loading standard opencv");
}
@@ -44,6 +41,16 @@ public class SomeCodes {
public static final double MB_threshold = 1024.0 * 1024.0;
public static final double GB_threshold = 1024.0 * 1024.0 * 1024.0;
private static void LoadLinuxLibrary(String library){
if (Platform.isLinux()){
File ff = new File(library);
if (ff.isFile()){
Logger.info("Loading library: "+library);
System.load(ff.getAbsolutePath());
}
}
}
@SuppressWarnings("resource")
public static String[] GetAudioFiles(){
try{
@@ -109,6 +116,13 @@ public class SomeCodes {
return defaultspeed;
}
public static boolean ValidIntArray(int[] xx){
if (xx!=null){
return xx.length>0;
}
return false;
}
public static boolean ValidInteger(String x){
try{
Integer.parseInt(x);
@@ -140,6 +154,25 @@ public class SomeCodes {
}
}
public static String MatToBase64(Mat mat){
if (mat!=null){
if (mat.cols()>0 && mat.rows()>0){
try{
BytePointer mob = new BytePointer();
opencv_imgcodecs.imencode(".jpg", mat, mob);
byte[] data = new byte[(int) mob.limit()];
mob.get(data);
String base64 = base64encoder.encodeToString(data);
mob.deallocate();
return base64;
} catch (Exception e){
Logger.error("Error converting Mat to Base64: "+e.getMessage());
}
}
}
return "";
}
public static String MatToBase64(org.opencv.core.Mat mat){
if (mat!=null){
if (mat.cols()>0 && mat.rows()>0){
@@ -157,6 +190,18 @@ public class SomeCodes {
return "";
}
public static Frame MatToFrame(Mat mat){
if (mat!=null && mat.cols()>0 && mat.rows()>0){
try{
return CoreMatConverter.convert(mat);
} catch (Exception e){
Logger.error("Error converting Mat to Frame: "+e.getMessage());
}
}
return null;
}
public static Frame MatToFrame(org.opencv.core.Mat mat){
if (mat!=null && mat.cols()>0 && mat.rows()>0){
try{
@@ -169,6 +214,56 @@ public class SomeCodes {
return null;
}
public static double[] getColumnValues(Mat mat, int colIndex, int[] rowIndices) {
if (mat.empty()) {
throw new IllegalArgumentException("Mat is empty.");
}
if (colIndex < 0 || colIndex >= mat.cols()) {
throw new IllegalArgumentException("Column index out of range.");
}
// Get the column as a new Mat
Mat colMat = mat.col(colIndex);
// Create an indexer for efficient access
DoubleIndexer indexer = colMat.createIndexer();
// Extract specific row values
double[] values = new double[rowIndices.length];
for (int i = 0; i < rowIndices.length; i++) {
values[i] = indexer.get(rowIndices[i], 0);
}
return values;
}
public static Point toPoint(Point2d p) {
return new Point((int)p.x(), (int)p.y());
}
public static Size toSize(Size2d s) {
return new Size((int)s.width(), (int)s.height());
}
public static Mat getSubmat(Mat mask, int colIndex, int startRow, int endRow) {
if (mask.empty()) {
throw new IllegalArgumentException("Mat is empty.");
}
if (colIndex < 0 || colIndex >= mask.cols()) {
throw new IllegalArgumentException("Column index out of range.");
}
if (startRow < 0 || endRow > mask.rows() || startRow >= endRow) {
throw new IllegalArgumentException("Invalid row range.");
}
// Extract column i
Mat colMat = mask.col(colIndex);
// Extract submatrix from row startRow to endRow
return colMat.rowRange(startRow, endRow);
}
/**
* Load properties file
@@ -252,5 +347,11 @@ public class SomeCodes {
}
}
public static GpuMat ConvertToGpuMat(Mat mat){
GpuMat gmat = new GpuMat();
gmat.upload(mat);
return gmat;
}
}

2
src/main/java/Web/WebServer.java
View File

@@ -269,9 +269,9 @@ public class WebServer {
connectedWebsocketClients.clear();
app.stop();
Logger.info("Web server stopped");
socketServer.stop();
socketIOClients.forEach((key, client) -> client.disconnect());
socketIOClients.clear();
socketServer.stop();
Logger.info("SocketIO server stopped");
} catch (JavalinException e){

87
src/main/java/id/co/gtc/Main.java
View File

@@ -2,7 +2,7 @@ package id.co.gtc;
import Audio.*;
import Camera.PanTiltController;
import Camera.RtspGrabber;
import Camera.RtspGrabber_opencv;
import Camera.VapixProtocol;
import Other.SomeCodes;
import SBC.*;
@@ -32,8 +32,9 @@ public class Main {
private static WebServer webServer;
//change parameter ini untuk menggunakan Yolo atau tidak
private static final boolean use_Yolo = false;
private static RtspGrabber rtspGrabber;
public static final boolean use_Yolo = false;
private static RtspGrabber_opencv rtspGrabber;
public static boolean haveCuda = false;
private static PanTiltController panTiltController;
private static VapixProtocol vapixProtocol;
@@ -54,7 +55,7 @@ public class Main {
private static ExecutorService gpioExecutor = null;
private static PCF8574 pcf8574 = null;
private static final String Version = "V1.0 (04/03/2025)";
private static final String Version = "V1.09 (17/03/2025)";
// Application start from here
public static void main(String[] args) {
System.setProperty("jna.debug_load", "false");
@@ -319,10 +320,7 @@ public class Main {
if (gpioExecutor!=null){
gpioExecutor.submit(()->{
pcf8574.SetPin(pin);
try {
Thread.sleep(20);
} catch (InterruptedException ignored) {
}
Sleep(20);
pcf8574.ClearPin(pin);
});
}
@@ -337,10 +335,7 @@ public class Main {
if (gpioExecutor!=null){
gpioExecutor.submit(()->{
GPIO.SetValue(pin, true);
try {
Thread.sleep(20);
} catch (InterruptedException ignored) {
}
Sleep(20);
GPIO.SetValue(pin, false);
});
}
@@ -474,6 +469,7 @@ public class Main {
if (ValidInteger(PanTiltID)){
panTiltController = new PanTiltController(portname, Integer.parseInt(baudrate), Integer.parseInt(PanTiltID));
//if (panTiltController.isOpen()) test_pantiltcontroller();
panTiltController.DisableMovementTest();
}
} else Logger.error("Invalid PTZ Baudrate");
} else Logger.error("Invalid PTZ Port");
@@ -484,24 +480,38 @@ public class Main {
// check if really activated
useOpenCL = opencv_core.useOpenCL();
Logger.info("OpenCL available={}, activated={}", haveOpenCL, useOpenCL);
int cudacount = opencv_core.getCudaEnabledDeviceCount();
if (cudacount>0){
Logger.info("CUDA enabled devices found: " + cudacount);
opencv_core.printCudaDeviceInfo(0);
haveCuda = opencv_core.getCudaEnabledDeviceCount()>0;
if (haveCuda){
Logger.info("CUDA enabled devices found: " + opencv_core.getCudaEnabledDeviceCount());
//opencv_core.printCudaDeviceInfo(0);
//Logger.info("CUDA enabled : "+opencv_core.useOptimized());
opencv_core.setUseOptimized(true);
Logger.info("CUDA enabled : "+opencv_core.useOptimized());
} else {
Logger.info("No CUDA enabled devices found");
}
// BytePointer buildinfo = opencv_core.getBuildInformation();
// if (buildinfo!=null){
// Logger.info("Opencv build information : {}", buildinfo.getString());
// } else{
// Logger.error("Failed to get OpenCV build information");
// }
Properties config = SomeCodes.LoadProperties("config.properties");
String targetip = config.getProperty("Camera_ip");
String rtsppath = config.getProperty("Camera_Rtsp_path");
int width = 1920;
int height = 1080;
// TODO test pakai sony camera, nanti hapus
targetip = "172.17.195.51";
rtsppath = "/video1";
width = 1280;
height = 720;
// test pakai sony camera, nanti hapus
// targetip = "172.17.195.51";
// rtsppath = "/video1";
// width = 1280;
// height = 720;
Logger.info("Camera IP: {}, Rtsp Path: {}, Width: {}, Height: {}", targetip, rtsppath, width,height);
rtspGrabber = null;
@@ -509,7 +519,7 @@ public class Main {
if (ValidString(rtsppath)){
if (IpIsReachable(targetip)){
Logger.info("Camera IP : "+targetip+" is reachable");
rtspGrabber = new RtspGrabber(targetip, rtsppath, use_Yolo);
rtspGrabber = new RtspGrabber_opencv(targetip, rtsppath);
rtspGrabber.Start(true, width, height);
} else Logger.warn("Camera IP : "+targetip+" is not reachable");
} else Logger.warn("Camera Path : "+rtsppath+" is not valid string");
@@ -585,23 +595,42 @@ public class Main {
switch (cmd){
// Pan Tilt Movement Commands
case "PAN LEFT" :
Logger.info("PAN LEFT");
speed = GetPanTiltSpeed(command.data, (byte)0x20);
if (panTiltController!=null) panTiltController.PanLeft(speed);
if (panTiltController!=null) {
panTiltController.PanLeft(speed);
}
return new WebsocketReply("PAN LEFT", String.valueOf(speed));
case "PAN RIGHT" :
Logger.info("PAN RIGHT");
speed = GetPanTiltSpeed(command.data, (byte)0x20);
if (panTiltController!=null) panTiltController.PanRight(speed);
if (panTiltController!=null) {
panTiltController.PanRight(speed);
}
return new WebsocketReply("PAN RIGHT", String.valueOf(speed));
case "TILT UP" :
Logger.info("TILT UP");
speed = GetPanTiltSpeed(command.data, (byte)0x20);
if (panTiltController!=null) panTiltController.TiltUp(speed);
// kebalik
//if (panTiltController!=null) panTiltController.TiltUp(speed);
if (panTiltController!=null) {
panTiltController.TiltDown(speed);
}
return new WebsocketReply("TILT UP", String.valueOf(speed));
case "TILT DOWN" :
Logger.info("TILT DOWN");
speed = GetPanTiltSpeed(command.data, (byte)0x20);
if (panTiltController!=null) panTiltController.TiltDown(speed);
// kebalik
//if (panTiltController!=null) panTiltController.TiltDown(speed);
if (panTiltController!=null) {
panTiltController.TiltUp(speed);
}
return new WebsocketReply("TILT DOWN", String.valueOf(speed));
case "STOP MOVEMENT" :
if (panTiltController!=null) panTiltController.StopMovement();
Logger.info("STOP MOVEMENT");
if (panTiltController!=null) {
panTiltController.StopMovement();
}
return new WebsocketReply("STOP MOVEMENT", "");
// Audio Related Commands
case "MUTE":
@@ -720,9 +749,9 @@ public class Main {
return switch (command.data) {
case "HQ" ->
new WebsocketReply("GET BASE64", "data:image/jpeg;base64," + rtspGrabber.getLastHQBase64(), rtspGrabber.HQStreamingStatus());
case "LQ" ->
new WebsocketReply("GET BASE64", "data:image/jpeg;base64," + rtspGrabber.getLastLQBase64(), rtspGrabber.LQStreamingStatus());
case "YOLO" ->
new WebsocketReply("GET BASE64", "data:image/jpeg;base64," + rtspGrabber.getLastLQBase64(), rtspGrabber.LQStreamingStatus());
case "LQ" ->
new WebsocketReply("GET BASE64", "data:image/jpeg;base64," + rtspGrabber.getLastYoloBase64(), rtspGrabber.LQStreamingStatus());
default -> new WebsocketReply("GET BASE64", "RTSP Grabber not initialized");
};