diff --git a/Config/config.properties b/Config/config.properties index 49cd969..43b47fa 100644 --- a/Config/config.properties +++ b/Config/config.properties @@ -4,6 +4,7 @@ Camera_port = 80 Camera_user = root Camera_password = password Camera_Rtsp_path = /axis-media/media.amp +#Camera_Rtsp_path = /video1 AudioFile01 = elangWav.wav AudioFile02 = gunshotsWav.wav AudioFile03 = pinkNoiseWav.wav diff --git a/config.properties b/config.properties index 0aac2f6..db8fb9e 100644 --- a/config.properties +++ b/config.properties @@ -2,10 +2,11 @@ AudioFile01=elangWav.wav AudioFile02=gunshotsWav.wav AudioFile03=pinkNoiseWav.wav -AudioFile04= +AudioFile04=null AudioFile05=null AudioVolumeOutput=100 Camera_Rtsp_path=/axis-media/media.amp +#Camera_Rtsp_path=/video1 Camera_ip=172.17.195.51 Camera_password=password Camera_port=80 diff --git a/onnx/yolo11n.onnx b/onnx/yolov8n.onnx similarity index 53% rename from onnx/yolo11n.onnx rename to onnx/yolov8n.onnx index cb0b371..7cb75f0 100644 Binary files a/onnx/yolo11n.onnx and b/onnx/yolov8n.onnx differ diff --git a/src/main/java/Audio/MultiUSBAudioPlayer.java b/src/main/java/Audio/MultiUSBAudioPlayer.java index a9b1107..af23591 100644 --- a/src/main/java/Audio/MultiUSBAudioPlayer.java +++ b/src/main/java/Audio/MultiUSBAudioPlayer.java @@ -72,6 +72,7 @@ public class MultiUSBAudioPlayer { List result = new ArrayList<>(); for(BassDeviceInfoWithId dev : playbackdevices){ if (dev.info.name.toLowerCase().contains(name.toLowerCase())){ + Logger.info("Found Device id={}, name={}", dev.id, dev.info.name); result.add(dev.id); } } @@ -98,7 +99,7 @@ public class MultiUSBAudioPlayer { validdevice.add(i); inited = true; } else { - Logger.error("Failed to initialize device {}", i); + Logger.error("Failed to initialize device {}, errorcode={}", i, bass.BASS_ErrorGetCode()); } } deviceid = validdevice.stream().mapToInt(i->i).toArray(); diff --git a/src/main/java/Camera/GrabbingTask.java b/src/main/java/Camera/GrabbingTask.java index a0520e2..3988fae 100644 --- a/src/main/java/Camera/GrabbingTask.java +++ b/src/main/java/Camera/GrabbingTask.java @@ -5,14 +5,14 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Consumer; import Other.SomeCodes; +import id.co.gtc.Main; import lombok.Getter; import lombok.Setter; import org.bytedeco.javacv.Frame; import org.bytedeco.javacv.FrameGrabber; import org.bytedeco.javacv.OpenCVFrameConverter; -import org.opencv.core.Mat; -import org.opencv.core.Size; import org.opencv.imgproc.Imgproc; +import org.opencv.core.*; import org.tinylog.Logger; public class GrabbingTask implements Runnable { @@ -39,8 +39,8 @@ public class GrabbingTask implements Runnable { // for FPS calculation private int intendedFps = 10; - private final boolean useYolo; - private final YoloDetector yolo; + private final YoloDetector_opencv yolo; + @SuppressWarnings("SameParameterValue") private void updateMessage(String message) { @@ -90,12 +90,13 @@ public class GrabbingTask implements Runnable { - public GrabbingTask(AtomicBoolean isGrabbing, FrameGrabber grabber, int fps, boolean useYolo) { + public GrabbingTask(AtomicBoolean isGrabbing, FrameGrabber grabber, int fps) { this.isGrabbing = isGrabbing; this.grabber = grabber; if (fps>0) intendedFps = fps; - this.useYolo = useYolo; - if (useYolo) yolo = new YoloDetector(); else yolo = null; + if (Main.use_Yolo) { + yolo = new YoloDetector_opencv(); + } else yolo = null; } @@ -104,28 +105,27 @@ public class GrabbingTask implements Runnable { } - private void processFrame(Frame fr){ + @SuppressWarnings({"CallToPrintStackTrace", "CatchMayIgnoreException"}) + private Mat processFrame(Frame fr){ if (fr!=null && fr.image!=null && fr.imageWidth>0 && fr.imageHeight>0){ try(var converter = new OpenCVFrameConverter.ToOrgOpenCvCoreMat()){ - org.opencv.core.Mat mat = converter.convert(fr); + Mat mat = converter.convert(fr); if (mat.cols()>0 && mat.rows()>0){ updateHQFrame(fr); updateHQBase64(SomeCodes.MatToBase64(mat)); Size lowsize = new Size(lowquality_width, lowquality_height); Mat resized = new Mat(); + Imgproc.resize(mat, resized, lowsize); + mat.release(); + if (resized.cols()>0 && resized.rows()>0){ - Frame resizedFrame = SomeCodes.CoreMatConverter.convert(resized); + Frame resizedFrame = converter.convert(resized); updateLQFrame(resizedFrame); updateLQBase64(SomeCodes.MatToBase64(resized)); - - if (useYolo){ - DetectYolo(resized); - } + return resized; } else Logger.error("processFrame resized size is 0"); - resized.release(); } else Logger.error("processFrame Mat size is 0"); - mat.release(); } catch (Exception e){ if (SomeCodes.ValidString(e.getMessage())) { if (!e.getMessage().startsWith("unknown exception")) { @@ -136,10 +136,12 @@ public class GrabbingTask implements Runnable { } - } else { - Logger.warn("processFrame have null frame"); - updateMessage("processFrame have null frame"); } +// else { +// Logger.warn("processFrame have null frame"); +// updateMessage("processFrame have null frame"); +// } + return null; } private void DetectYolo(Mat mat){ @@ -175,26 +177,31 @@ public class GrabbingTask implements Runnable { } } + @SuppressWarnings("CallToPrintStackTrace") @Override public void run() { isGrabbing.set(true); Logger.info("Grabbing Task started"); double fps = grabber.getFrameRate(); int skippedframes = (int)(fps / intendedFps); - //Logger.info("Grabber framerate = {}, intendedFps = {}, Skipping frames = {}", fps, intendedFps, skippedframes); + Logger.info("Grabber framerate = {}, intendedFps = {}, Skipping frames = {}", fps, intendedFps, skippedframes); int framecount = 0; flush_grabber(); long starttick = System.currentTimeMillis(); while (isGrabbing.get()) { try{ Thread.yield(); + Frame frame = grabber.grab(); if (framecount0) { int xx = (int) (1000 / elapsed); - if (xx0x3F) speed = 0x3F; - byte[] command = new byte[]{0, cameraid, 0, 8, speed, 0, 0}; + byte[] command = new byte[]{0, cameraid, 0, 8, 0, speed, 0}; command[6] = Checksum(command); // add checksum command[0] = (byte) 0xFF; // add synchronization byte if (isOpen()) { Main.Max485Direction(true); - serialPort.writeBytes(command, command.length); + int written = serialPort.writeBytes(command, command.length); Main.Max485Direction(false); Main.Blink_LedPanTilt(); + Logger.info("Tilt Up written {} bytes", written); } else Logger.warn("Tilt Up failed, serial port not open"); } @@ -141,17 +145,44 @@ public class PanTiltController { if (speed<0) speed = 0; if (speed>0x3F) speed = 0x3F; - byte[] command = new byte[]{0, cameraid, 0, 16, speed, 0, 0}; + byte[] command = new byte[]{0, cameraid, 0, 16, 0, speed, 0}; command[6] = Checksum(command); // add checksum command[0] = (byte) 0xFF; // add synchronization byte if (isOpen()) { Main.Max485Direction(true); - serialPort.writeBytes(command, command.length); + int written = serialPort.writeBytes(command, command.length); Main.Max485Direction(false); Main.Blink_LedPanTilt(); + Logger.info("Tilt Down written {} bytes", written); } else Logger.warn("Tilt Down failed, serial port not open"); } + public void DisableMovementTest(){ + byte[] command = new byte[]{(byte)0xFF, cameraid, 0, 7, 0, (byte)0x54, (byte)0x5c}; + //command[6] = Checksum(command); + //command[0] = (byte) 0xFF; + if (isOpen()) { + Main.Max485Direction(true); + int written = serialPort.writeBytes(command, command.length); + Main.Max485Direction(false); + Main.Blink_LedPanTilt(); + Logger.info("DisableMovement test written {} bytes", written); + } else Logger.warn("DisableMovement failed, serial port not open"); + } + + public void EnableMovementTest(){ + byte[] command = new byte[]{(byte)0xFF, cameraid, 0, 3, 0, (byte)0x54, (byte) 0x58}; + //command[6] = Checksum(command); + //command[0] = (byte) 0xFF; + if (isOpen()) { + Main.Max485Direction(true); + int written = serialPort.writeBytes(command, command.length); + Main.Max485Direction(false); + Main.Blink_LedPanTilt(); + Logger.info("EnableMovement test written {} bytes", written); + } else Logger.warn("EnableMovement failed, serial port not open"); + } + /** * Go to Preset * @param preset preset number diff --git a/src/main/java/Camera/RtspGrabber.java b/src/main/java/Camera/RtspGrabber.java index 40cf6d7..b4f32da 100644 --- a/src/main/java/Camera/RtspGrabber.java +++ b/src/main/java/Camera/RtspGrabber.java @@ -3,7 +3,6 @@ import lombok.Getter; import org.bytedeco.ffmpeg.global.avutil; import org.bytedeco.javacv.FFmpegFrameGrabber; import org.bytedeco.javacv.Frame; -import org.bytedeco.opencv.global.opencv_core; import org.jetbrains.annotations.NotNull; import org.tinylog.Logger; @@ -26,11 +25,9 @@ public class RtspGrabber { private @Getter int HQHeight = 0; private @Getter int LQWidth = 0; private @Getter int LQHeight = 0; - private final boolean useYolo; private GrabbingTask grabbingTask; - public RtspGrabber(String ip, String path, boolean useYolo) { - this.useYolo = useYolo; + public RtspGrabber(String ip, String path) { this.rtspUrl = "rtsp://" + ip + path; Logger.info("RtspGrabber created with url: " + rtspUrl); @@ -142,7 +139,7 @@ public class RtspGrabber { @NotNull private GrabbingTask getGrabbingTask() { - GrabbingTask tt = new GrabbingTask(isGrabbing, grabber,10, useYolo); + GrabbingTask tt = new GrabbingTask(isGrabbing, grabber,10); tt.setOnMessageUpdate(Logger::info); tt.setOnHQFrameUpdate(value -> { if (value!=null){ diff --git a/src/main/java/Camera/RtspGrabber_opencv.java b/src/main/java/Camera/RtspGrabber_opencv.java new file mode 100644 index 0000000..4f4a984 --- /dev/null +++ b/src/main/java/Camera/RtspGrabber_opencv.java @@ -0,0 +1,169 @@ +package Camera; +import lombok.Getter; +import org.bytedeco.ffmpeg.global.avutil; +import org.bytedeco.javacv.FFmpegFrameGrabber; +import org.bytedeco.javacv.Frame; +import org.jetbrains.annotations.NotNull; +import org.tinylog.Logger; + +import java.util.concurrent.atomic.AtomicBoolean; + +import static Other.SomeCodes.gson; + +@SuppressWarnings("unused") +public class RtspGrabber_opencv { + private final String rtspUrl; + private FFmpegFrameGrabber grabber; + private final AtomicBoolean isGrabbing = new AtomicBoolean(false); + private Frame lastHQFrame = null; + private Frame lastLQFrame = null; + private Frame lastYoloFrame = null; + private String lastHQBase64 = null; + private String lastLQBase64 = null; + private String lastYoloBase64 = null; + private @Getter int HQWidth = 0; + private @Getter int HQHeight = 0; + private @Getter int LQWidth = 0; + private @Getter int LQHeight = 0; + private GrabbingTask grabbingTask; + + public RtspGrabber_opencv(String ip, String path) { + + this.rtspUrl = "rtsp://" + ip + path; + Logger.info("RtspGrabber created with url: " + rtspUrl); + } + + private synchronized void setLastHQFrame(Frame frame){ + lastHQFrame = frame; + } + + public synchronized Frame getLastHQFrame(){ + return lastHQFrame; + } + + private synchronized void setLastLQFrame(Frame frame){ + lastLQFrame = frame; + } + + public synchronized Frame getLastLQFrame(){ + return lastLQFrame; + } + + private synchronized void setLastHQBase64(String base64){ + lastHQBase64 = base64; + } + + public synchronized String getLastHQBase64(){ + return lastHQBase64; + } + + private synchronized void setLastLQBase64(String base64){ + lastLQBase64 = base64; + } + + public synchronized String getLastLQBase64(){ + return lastLQBase64; + } + + private synchronized void setLastYoloBase64(String base64){ + lastYoloBase64 = base64; + } + + public synchronized String getLastYoloBase64(){ + return lastYoloBase64; + } + + private synchronized void setLastYoloFrame(Frame frame){ + lastYoloFrame = frame; + } + + public synchronized Frame getLastYoloFrame(){ + return lastYoloFrame; + } + + /** + * Start grabbing frames from rtsp + * @param useTcp Use tcp instead of udp + */ + public void Start(boolean useTcp, final int width, final int height){ + + try{ + grabber = FFmpegFrameGrabber.createDefault(rtspUrl); + if (useTcp) grabber.setOption("rtsp_transport", "tcp"); + + grabber.setPixelFormat(avutil.AV_PIX_FMT_BGR24); + grabber.start(); + + avutil.av_log_set_level(avutil.AV_LOG_ERROR); + + Logger.info("Grabber started"); + GrabbingTask tt = getGrabbingTask(); + grabbingTask = tt; + new Thread(tt).start(); + + } catch (Exception e){ + Logger.error("Error starting grabber: " + e.getMessage()); + } + } + + + + /** + * Stop grabbing frames + */ + public void Stop(){ + isGrabbing.set(false); + if (grabbingTask!=null){ + grabbingTask.Stop(); + } + grabbingTask = null; + if (grabber!=null) { + try{ + grabber.stop(); + grabber.releaseUnsafe(); + Logger.info("Grabber stopped"); + } catch (Exception e){ + Logger.error("Error stopping grabber: " + e.getMessage()); + } + } + grabber = null; + } + + public String LQStreamingStatus(){ + return gson.toJson(new String[]{String.valueOf(LQWidth), String.valueOf(LQHeight) , String.valueOf(grabbingTask!=null ? grabbingTask.getCaptureFPS():"0")}); + } + + public String HQStreamingStatus(){ + return gson.toJson(new String[]{String.valueOf(HQWidth), String.valueOf(HQHeight) , String.valueOf(grabbingTask!=null ? grabbingTask.getCaptureFPS(): "0")}); + } + + @NotNull + private GrabbingTask getGrabbingTask() { + GrabbingTask tt = new GrabbingTask(isGrabbing, grabber,5); + tt.setOnMessageUpdate(Logger::info); + tt.setOnHQFrameUpdate(value -> { + if (value!=null){ + if (value.imageWidth>0 && value.imageHeight>0){ + setLastHQFrame(value); + HQWidth = value.imageWidth; + HQHeight = value.imageHeight; + } + + } + }); + tt.setOnLQFrameUpdate(value -> { + if (value!=null){ + if (value.imageWidth>0 && value.imageHeight>0){ + setLastLQFrame(value); + LQWidth = value.imageWidth; + LQHeight = value.imageHeight; + } + } + }); + tt.setOnHQBase64Update(this::setLastHQBase64); + tt.setOnLQBase64Update(this::setLastLQBase64); + tt.setOnYoloUpdate(this::setLastYoloFrame); + tt.setOnYoloBase64Update(this::setLastYoloBase64); + return tt; + } +} diff --git a/src/main/java/Camera/YoloDetector.java b/src/main/java/Camera/YoloDetector.java index c14f5b8..4dcbfb6 100644 --- a/src/main/java/Camera/YoloDetector.java +++ b/src/main/java/Camera/YoloDetector.java @@ -1,11 +1,17 @@ package Camera; import Other.SomeCodes; +import id.co.gtc.Main; import lombok.Getter; -import org.opencv.core.*; -import org.opencv.dnn.Dnn; -import org.opencv.dnn.Net; +import org.bytedeco.javacpp.DoublePointer; +import org.bytedeco.javacpp.FloatPointer; +import org.bytedeco.javacpp.IntPointer; +import org.bytedeco.opencv.global.opencv_core; +import org.bytedeco.opencv.global.opencv_dnn; +import org.bytedeco.opencv.global.opencv_imgproc; +import org.bytedeco.opencv.opencv_core.*; +import org.bytedeco.opencv.opencv_dnn.Net; import org.opencv.imgproc.Imgproc; import org.tinylog.Logger; @@ -22,47 +28,46 @@ public class YoloDetector { private @Getter boolean NetLoaded; private List classes ; - // minimum confidence for detections - private final float confidence_threshold = 0.4f; - // Non-maximum suppression threshold - private final float nms_threshold = 0.4f; - public YoloDetector() { NetLoaded = false; net = null; classes = null; - String onnxfile = SomeCodes.ExtractResource("/yolo11n.onnx", SomeCodes.currentDirectory); String namesfile = SomeCodes.ExtractResource("/coco.names", SomeCodes.currentDirectory); - if (SomeCodes.ValidFile(onnxfile)){ - if (SomeCodes.ValidFile(namesfile)){ - net = Dnn.readNetFromONNX(onnxfile); - net.setPreferableBackend(Dnn.DNN_BACKEND_CUDA); - net.setPreferableTarget(Dnn.DNN_TARGET_CUDA); - //net.setPreferableBackend(Dnn.DNN_BACKEND_OPENCV); - //net.setPreferableBackend(Dnn.DNN_BACKEND_INFERENCE_ENGINE); - //net.setPreferableTarget(Dnn.DNN_TARGET_CPU); - //net.setPreferableTarget(Dnn.DNN_TARGET_NPU); + String netfile = SomeCodes.ExtractResource("/yolov8n.onnx", SomeCodes.currentDirectory); + if (SomeCodes.ValidFile(netfile)){ + net = opencv_dnn.readNetFromONNX(netfile); + if (Main.haveCuda){ + net.setPreferableBackend(opencv_dnn.DNN_BACKEND_CUDA); + net.setPreferableTarget(opencv_dnn.DNN_TARGET_CUDA_FP16); + Logger.info("Net loaded with CUDA"); + } else { + net.setPreferableBackend(opencv_dnn.DNN_BACKEND_OPENCV); + net.setPreferableTarget(opencv_dnn.DNN_TARGET_CPU); + Logger.info("Net loaded with CPU"); + } + } else Logger.error("net file not found"); - classes = new ArrayList<>(); - if (namesfile!=null){ - try(BufferedReader br = new BufferedReader(new FileReader(namesfile))) { - String line; - while ((line = br.readLine()) != null) { - classes.add(line); - } - } catch (Exception e){ - Logger.error("reading names file, Exception : ",e.getMessage()); + if (net!=null){ + classes = new ArrayList<>(); + if (namesfile!=null){ + try(BufferedReader br = new BufferedReader(new FileReader(namesfile))) { + String line; + while ((line = br.readLine()) != null) { + classes.add(line); } + } catch (Exception e){ + Logger.error("reading names file, Exception : ",e.getMessage()); } + } + + if (!net.empty()){ + if (!classes.isEmpty()){ + NetLoaded = true; + } else Logger.error("names file is empty"); + } else Logger.error("net is empty"); + } else Logger.error("Net is not loaded"); - if (!net.empty()){ - if (!classes.isEmpty()){ - NetLoaded = true; - } else Logger.error("names file is empty"); - } else Logger.error("net is empty"); - } else Logger.error("names file not found"); - } else Logger.error("onnx file not found"); } // Source : https://blog.csdn.net/taoli188/article/details/134720614 @@ -71,12 +76,13 @@ public class YoloDetector { if (NetLoaded){ Size inputSize = new Size(640, 640); Scalar mean = new Scalar(0, 0, 0, 0); - float scaleFactor = 1.0f/255.0f; + double scaleFactor = 1.0f/255.0f; boolean swapRB = true; boolean crop = false; - Mat blob = Dnn.blobFromImage(frame, scaleFactor, inputSize, mean , swapRB, crop, CV_32F); - //Mat blob = Dnn.blobFromImage(frame, scaleFactor, inputSize); + //Mat blob = opencv_dnn.blobFromImage(frame, scaleFactor, inputSize, mean , swapRB, crop, CV_32F); + Mat blob = new Mat(); + opencv_dnn.blobFromImage(frame, blob, scaleFactor, inputSize, mean, swapRB, crop, CV_32F); net.setInput(blob); Mat predict = net.forward(); @@ -94,8 +100,10 @@ public class YoloDetector { } + @SuppressWarnings("CallToPrintStackTrace") public Mat Process(Mat frame, Mat predict, Mat mask){ try{ + double width = frame.cols() / 640.0; double height = frame.rows() / 640.0; Rect2d[] rect2d = new Rect2d[mask.cols()]; @@ -103,28 +111,48 @@ public class YoloDetector { int[] classid = new int[mask.cols()]; for(int i=0;i0){ + int[] idx = new int[(int)indices.limit()]; + indices.get(idx); for(int ii : idx){ - Rect rr = new Rect(rect2d[ii].tl(), rect2d[ii].size()); - Imgproc.rectangle(frame, rr, new Scalar(0, 255, 0), 2); - Imgproc.putText(frame, classes.get(classid[ii])+" "+String.format("%.2f", scoref[ii]), rr.tl(), Imgproc.FONT_HERSHEY_SIMPLEX, 1.0, new Scalar(0, 0, 255)); + Rect rr = new Rect(SomeCodes.toPoint(rect2d[ii].tl()), SomeCodes.toSize(rect2d[ii].size())); + opencv_imgproc.rectangle(frame,rr, Scalar.GREEN); + opencv_imgproc.putText(frame, classes.get(classid[ii])+" "+String.format("%.2f", scoref[ii]), rr.tl(), Imgproc.FONT_HERSHEY_SIMPLEX, 1.0, Scalar.RED); } } @@ -138,87 +166,4 @@ public class YoloDetector { return frame; } - public Mat processDetection(Mat frame, Mat output){ - int rows = output.rows(); - int cols = output.cols(); - List boxes = new ArrayList<>(); - List confidences = new ArrayList<>(); - List classIds = new ArrayList<>(); - - for (int i = 0; i < rows; i++){ - Mat row = output.row(i); - Mat scores = row.colRange(5, cols); - Core.MinMaxLocResult mm = Core.minMaxLoc(scores); - float confidence = (float)mm.maxVal; - Point classIdPoint = mm.maxLoc; - if (confidence>confidence_threshold){ - int centerX = (int)(row.get(0, 0)[0] * frame.cols()); - int centerY = (int)(row.get(0, 1)[0] * frame.rows()); - int width = (int)(row.get(0, 2)[0] * frame.cols()); - int height = (int)(row.get(0, 3)[0] * frame.rows()); - int left = centerX - width / 2; - int top = centerY - height / 2; - boxes.add(new Rect(left, top, width, height)); - confidences.add(confidence); - classIds.add((int)classIdPoint.x); - } - } - - MatOfRect2d rects = new MatOfRect2d(ToRect2D(boxes)); - MatOfFloat conf = new MatOfFloat(ToFloatArray(confidences)); - MatOfInt ids = new MatOfInt(); - - Dnn.NMSBoxes(rects, conf, confidence_threshold, nms_threshold, ids); - - int[] indices = ids.toArray(); - if (indices.length>0){ - for (int idx : indices){ - - Rect box = boxes.get(idx); - int classId = classIds.get(idx); - float confidence = confidences.get(idx); - String label = classes.get(classId); - Logger.info("Detect ["+idx+"] ClassID: "+classId+" Label: "+label+" Confidence: "+confidence); - - Imgproc.rectangle(frame, box, new Scalar(0, 255, 0), 2); - String text = label + " " + String.format("%.2f", confidence); - int[] baseLine = new int[1]; - Size labelSize = Imgproc.getTextSize(text, Imgproc.FONT_HERSHEY_SIMPLEX, 0.5, 1, baseLine); - Imgproc.rectangle(frame, new Point(box.x, box.y - labelSize.height), - new Point(box.x + labelSize.width, box.y + baseLine[0]), - new Scalar(255, 255, 255), Imgproc.FILLED); - Imgproc.putText(frame, text, new Point(box.x, box.y), - Imgproc.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar(0, 0, 0)); - } - } - - return frame; - } - - private Rect2d[] ToRect2D(List list){ - if (list!=null && list.size()>0){ - Rect2d array[] = new Rect2d[list.size()]; - for (int i = 0; i < list.size(); i++){ - Rect r = list.get(i); - array[i] = new Rect2d(r.x, r.y, r.width, r.height); - } - return array; - } - return new Rect2d[0]; - } - - private float[] ToFloatArray(List list){ - if (list!=null && list.size()>0){ - float array[] = new float[list.size()]; - for (int i = 0; i < list.size(); i++){ - array[i] = list.get(i); - } - return array; - } - return new float[0]; - } - - - - } diff --git a/src/main/java/Camera/YoloDetector_opencv.java b/src/main/java/Camera/YoloDetector_opencv.java new file mode 100644 index 0000000..753601c --- /dev/null +++ b/src/main/java/Camera/YoloDetector_opencv.java @@ -0,0 +1,146 @@ +package Camera; + +import Other.SomeCodes; +import id.co.gtc.Main; +import lombok.Getter; +import org.opencv.core.*; +import org.opencv.dnn.Dnn; +import org.opencv.dnn.Net; +import org.opencv.imgproc.Imgproc; +import org.tinylog.Logger; + +import java.io.BufferedReader; +import java.io.FileReader; +import java.util.ArrayList; +import java.util.List; + +import static org.bytedeco.opencv.global.opencv_core.CV_32F; + +public class YoloDetector_opencv { + private Net net; + private @Getter boolean NetLoaded; + private List classes ; + + + public YoloDetector_opencv() { + NetLoaded = false; + net = null; + classes = null; + String namesfile = SomeCodes.ExtractResource("/coco.names", SomeCodes.currentDirectory); + String netfile = SomeCodes.ExtractResource("/yolov8n.onnx", SomeCodes.currentDirectory); + if (SomeCodes.ValidFile(netfile)){ + net = Dnn.readNetFromONNX(netfile); + if (Main.haveCuda){ + net.setPreferableBackend(Dnn.DNN_BACKEND_CUDA); + net.setPreferableTarget(Dnn.DNN_TARGET_CUDA_FP16); + Logger.info("Net loaded with CUDA"); + } else { + net.setPreferableBackend(Dnn.DNN_BACKEND_OPENCV); + net.setPreferableTarget(Dnn.DNN_TARGET_CPU); + Logger.info("Net loaded with CPU"); + } + } else Logger.error("net file not found"); + + if (net!=null){ + classes = new ArrayList<>(); + if (namesfile!=null){ + try(BufferedReader br = new BufferedReader(new FileReader(namesfile))) { + String line; + while ((line = br.readLine()) != null) { + classes.add(line); + } + } catch (Exception e){ + Logger.error("reading names file, Exception : ",e.getMessage()); + } + } + + if (!net.empty()){ + if (!classes.isEmpty()){ + NetLoaded = true; + } else Logger.error("names file is empty"); + } else Logger.error("net is empty"); + } else Logger.error("Net is not loaded"); + + } + + // Source : https://blog.csdn.net/taoli188/article/details/134720614 + public Mat[] Detect(Mat frame){ + try{ + if (NetLoaded){ + Size inputSize = new Size(640, 640); + Scalar mean = new Scalar(0, 0, 0, 0); + float scaleFactor = 1.0f/255.0f; + boolean swapRB = true; + boolean crop = false; + + Mat blob = Dnn.blobFromImage(frame, scaleFactor, inputSize, mean , swapRB, crop, CV_32F); + //Mat blob = Dnn.blobFromImage(frame, scaleFactor, inputSize); + + net.setInput(blob); + + Mat predict = net.forward(); + Mat mask = predict.reshape(0,1).reshape(0, predict.size(1)); + + return new Mat[]{predict,mask}; + } else { + Logger.error("Net not loaded"); + return new Mat[0]; + } + } catch (Exception e){ + Logger.error("Error detecting: "+e.getMessage()); + return new Mat[0]; + } + + } + + @SuppressWarnings("CallToPrintStackTrace") + public Mat Process(Mat frame, Mat predict, Mat mask){ + try{ + + double width = frame.cols() / 640.0; + double height = frame.rows() / 640.0; + Rect2d[] rect2d = new Rect2d[mask.cols()]; + float[] scoref = new float[mask.cols()]; + int[] classid = new int[mask.cols()]; + + for(int i=0;i0; + } + return false; + } + public static boolean ValidInteger(String x){ try{ Integer.parseInt(x); @@ -140,6 +154,25 @@ public class SomeCodes { } } + public static String MatToBase64(Mat mat){ + if (mat!=null){ + if (mat.cols()>0 && mat.rows()>0){ + try{ + BytePointer mob = new BytePointer(); + opencv_imgcodecs.imencode(".jpg", mat, mob); + byte[] data = new byte[(int) mob.limit()]; + mob.get(data); + String base64 = base64encoder.encodeToString(data); + mob.deallocate(); + return base64; + } catch (Exception e){ + Logger.error("Error converting Mat to Base64: "+e.getMessage()); + } + } + } + return ""; + } + public static String MatToBase64(org.opencv.core.Mat mat){ if (mat!=null){ if (mat.cols()>0 && mat.rows()>0){ @@ -157,6 +190,18 @@ public class SomeCodes { return ""; } + public static Frame MatToFrame(Mat mat){ + if (mat!=null && mat.cols()>0 && mat.rows()>0){ + try{ + return CoreMatConverter.convert(mat); + } catch (Exception e){ + Logger.error("Error converting Mat to Frame: "+e.getMessage()); + } + } + + return null; + } + public static Frame MatToFrame(org.opencv.core.Mat mat){ if (mat!=null && mat.cols()>0 && mat.rows()>0){ try{ @@ -169,6 +214,56 @@ public class SomeCodes { return null; } + public static double[] getColumnValues(Mat mat, int colIndex, int[] rowIndices) { + if (mat.empty()) { + throw new IllegalArgumentException("Mat is empty."); + } + if (colIndex < 0 || colIndex >= mat.cols()) { + throw new IllegalArgumentException("Column index out of range."); + } + + // Get the column as a new Mat + Mat colMat = mat.col(colIndex); + + // Create an indexer for efficient access + DoubleIndexer indexer = colMat.createIndexer(); + + // Extract specific row values + double[] values = new double[rowIndices.length]; + for (int i = 0; i < rowIndices.length; i++) { + values[i] = indexer.get(rowIndices[i], 0); + } + + return values; + } + + + + public static Point toPoint(Point2d p) { + return new Point((int)p.x(), (int)p.y()); + } + + public static Size toSize(Size2d s) { + return new Size((int)s.width(), (int)s.height()); + } + + public static Mat getSubmat(Mat mask, int colIndex, int startRow, int endRow) { + if (mask.empty()) { + throw new IllegalArgumentException("Mat is empty."); + } + if (colIndex < 0 || colIndex >= mask.cols()) { + throw new IllegalArgumentException("Column index out of range."); + } + if (startRow < 0 || endRow > mask.rows() || startRow >= endRow) { + throw new IllegalArgumentException("Invalid row range."); + } + + // Extract column i + Mat colMat = mask.col(colIndex); + + // Extract submatrix from row startRow to endRow + return colMat.rowRange(startRow, endRow); + } /** * Load properties file @@ -252,5 +347,11 @@ public class SomeCodes { } } + public static GpuMat ConvertToGpuMat(Mat mat){ + GpuMat gmat = new GpuMat(); + gmat.upload(mat); + return gmat; + } + } diff --git a/src/main/java/Web/WebServer.java b/src/main/java/Web/WebServer.java index 2ec914f..52860b2 100644 --- a/src/main/java/Web/WebServer.java +++ b/src/main/java/Web/WebServer.java @@ -269,9 +269,9 @@ public class WebServer { connectedWebsocketClients.clear(); app.stop(); Logger.info("Web server stopped"); + socketServer.stop(); socketIOClients.forEach((key, client) -> client.disconnect()); socketIOClients.clear(); - socketServer.stop(); Logger.info("SocketIO server stopped"); } catch (JavalinException e){ diff --git a/src/main/java/id/co/gtc/Main.java b/src/main/java/id/co/gtc/Main.java index d712d86..21da565 100644 --- a/src/main/java/id/co/gtc/Main.java +++ b/src/main/java/id/co/gtc/Main.java @@ -2,7 +2,7 @@ package id.co.gtc; import Audio.*; import Camera.PanTiltController; -import Camera.RtspGrabber; +import Camera.RtspGrabber_opencv; import Camera.VapixProtocol; import Other.SomeCodes; import SBC.*; @@ -32,8 +32,9 @@ public class Main { private static WebServer webServer; //change parameter ini untuk menggunakan Yolo atau tidak - private static final boolean use_Yolo = false; - private static RtspGrabber rtspGrabber; + public static final boolean use_Yolo = false; + private static RtspGrabber_opencv rtspGrabber; + public static boolean haveCuda = false; private static PanTiltController panTiltController; private static VapixProtocol vapixProtocol; @@ -54,7 +55,7 @@ public class Main { private static ExecutorService gpioExecutor = null; private static PCF8574 pcf8574 = null; - private static final String Version = "V1.0 (04/03/2025)"; + private static final String Version = "V1.09 (17/03/2025)"; // Application start from here public static void main(String[] args) { System.setProperty("jna.debug_load", "false"); @@ -319,10 +320,7 @@ public class Main { if (gpioExecutor!=null){ gpioExecutor.submit(()->{ pcf8574.SetPin(pin); - try { - Thread.sleep(20); - } catch (InterruptedException ignored) { - } + Sleep(20); pcf8574.ClearPin(pin); }); } @@ -337,10 +335,7 @@ public class Main { if (gpioExecutor!=null){ gpioExecutor.submit(()->{ GPIO.SetValue(pin, true); - try { - Thread.sleep(20); - } catch (InterruptedException ignored) { - } + Sleep(20); GPIO.SetValue(pin, false); }); } @@ -474,6 +469,7 @@ public class Main { if (ValidInteger(PanTiltID)){ panTiltController = new PanTiltController(portname, Integer.parseInt(baudrate), Integer.parseInt(PanTiltID)); //if (panTiltController.isOpen()) test_pantiltcontroller(); + panTiltController.DisableMovementTest(); } } else Logger.error("Invalid PTZ Baudrate"); } else Logger.error("Invalid PTZ Port"); @@ -484,24 +480,38 @@ public class Main { // check if really activated useOpenCL = opencv_core.useOpenCL(); Logger.info("OpenCL available={}, activated={}", haveOpenCL, useOpenCL); - int cudacount = opencv_core.getCudaEnabledDeviceCount(); - if (cudacount>0){ - Logger.info("CUDA enabled devices found: " + cudacount); - opencv_core.printCudaDeviceInfo(0); + haveCuda = opencv_core.getCudaEnabledDeviceCount()>0; + if (haveCuda){ + Logger.info("CUDA enabled devices found: " + opencv_core.getCudaEnabledDeviceCount()); + //opencv_core.printCudaDeviceInfo(0); + //Logger.info("CUDA enabled : "+opencv_core.useOptimized()); + opencv_core.setUseOptimized(true); + Logger.info("CUDA enabled : "+opencv_core.useOptimized()); + } else { Logger.info("No CUDA enabled devices found"); } +// BytePointer buildinfo = opencv_core.getBuildInformation(); +// if (buildinfo!=null){ +// Logger.info("Opencv build information : {}", buildinfo.getString()); +// } else{ +// Logger.error("Failed to get OpenCV build information"); +// } + + + + Properties config = SomeCodes.LoadProperties("config.properties"); String targetip = config.getProperty("Camera_ip"); String rtsppath = config.getProperty("Camera_Rtsp_path"); int width = 1920; int height = 1080; - // TODO test pakai sony camera, nanti hapus - targetip = "172.17.195.51"; - rtsppath = "/video1"; - width = 1280; - height = 720; + // test pakai sony camera, nanti hapus +// targetip = "172.17.195.51"; +// rtsppath = "/video1"; +// width = 1280; +// height = 720; Logger.info("Camera IP: {}, Rtsp Path: {}, Width: {}, Height: {}", targetip, rtsppath, width,height); rtspGrabber = null; @@ -509,7 +519,7 @@ public class Main { if (ValidString(rtsppath)){ if (IpIsReachable(targetip)){ Logger.info("Camera IP : "+targetip+" is reachable"); - rtspGrabber = new RtspGrabber(targetip, rtsppath, use_Yolo); + rtspGrabber = new RtspGrabber_opencv(targetip, rtsppath); rtspGrabber.Start(true, width, height); } else Logger.warn("Camera IP : "+targetip+" is not reachable"); } else Logger.warn("Camera Path : "+rtsppath+" is not valid string"); @@ -585,23 +595,42 @@ public class Main { switch (cmd){ // Pan Tilt Movement Commands case "PAN LEFT" : + Logger.info("PAN LEFT"); speed = GetPanTiltSpeed(command.data, (byte)0x20); - if (panTiltController!=null) panTiltController.PanLeft(speed); + if (panTiltController!=null) { + panTiltController.PanLeft(speed); + } return new WebsocketReply("PAN LEFT", String.valueOf(speed)); case "PAN RIGHT" : + Logger.info("PAN RIGHT"); speed = GetPanTiltSpeed(command.data, (byte)0x20); - if (panTiltController!=null) panTiltController.PanRight(speed); + if (panTiltController!=null) { + panTiltController.PanRight(speed); + } return new WebsocketReply("PAN RIGHT", String.valueOf(speed)); case "TILT UP" : + Logger.info("TILT UP"); speed = GetPanTiltSpeed(command.data, (byte)0x20); - if (panTiltController!=null) panTiltController.TiltUp(speed); + // kebalik + //if (panTiltController!=null) panTiltController.TiltUp(speed); + if (panTiltController!=null) { + panTiltController.TiltDown(speed); + } return new WebsocketReply("TILT UP", String.valueOf(speed)); case "TILT DOWN" : + Logger.info("TILT DOWN"); speed = GetPanTiltSpeed(command.data, (byte)0x20); - if (panTiltController!=null) panTiltController.TiltDown(speed); + // kebalik + //if (panTiltController!=null) panTiltController.TiltDown(speed); + if (panTiltController!=null) { + panTiltController.TiltUp(speed); + } return new WebsocketReply("TILT DOWN", String.valueOf(speed)); case "STOP MOVEMENT" : - if (panTiltController!=null) panTiltController.StopMovement(); + Logger.info("STOP MOVEMENT"); + if (panTiltController!=null) { + panTiltController.StopMovement(); + } return new WebsocketReply("STOP MOVEMENT", ""); // Audio Related Commands case "MUTE": @@ -720,9 +749,9 @@ public class Main { return switch (command.data) { case "HQ" -> new WebsocketReply("GET BASE64", "data:image/jpeg;base64," + rtspGrabber.getLastHQBase64(), rtspGrabber.HQStreamingStatus()); - case "LQ" -> - new WebsocketReply("GET BASE64", "data:image/jpeg;base64," + rtspGrabber.getLastLQBase64(), rtspGrabber.LQStreamingStatus()); case "YOLO" -> + new WebsocketReply("GET BASE64", "data:image/jpeg;base64," + rtspGrabber.getLastLQBase64(), rtspGrabber.LQStreamingStatus()); + case "LQ" -> new WebsocketReply("GET BASE64", "data:image/jpeg;base64," + rtspGrabber.getLastYoloBase64(), rtspGrabber.LQStreamingStatus()); default -> new WebsocketReply("GET BASE64", "RTSP Grabber not initialized"); };