0.0.1.6
This commit is contained in:
1 parent
97649d73eb
commit
e77afd4d29
127 files changed
+7320
-1886
No files matched your search
@@ -1,4 +1,4 @@
|
||||
#Tue Sep 29 23:22:28 CEST 2026
|
||||
display=\:0
|
||||
process-id=7
|
||||
user=kotama
|
||||
#Sat Oct 10 23:02:27 CEST 2026
|
||||
host=fedora
|
||||
process-id=3
|
||||
user=Kotama_Chio
|
||||
+305
@@ -2119,3 +2119,308 @@ Caused by: java.net.UnknownHostException: CelesteKC-PC: Name or service not know
|
||||
|
||||
!ENTRY ch.qos.logback.classic 1 0 2026-09-29 23:22:28.792
|
||||
!MESSAGE Logback config file: /home/kotama/Silly-Jirafe/.metadata/.plugins/org.eclipse.m2e.logback/logback.2.7.101.20251017-1242.xml
|
||||
!SESSION 2026-10-04 17:06:10.851 -----------------------------------------------
|
||||
eclipse.buildId=4.41.0.20260903-0719
|
||||
java.version=25.0.4.1
|
||||
java.vendor=Eclipse Adoptium
|
||||
BootLoader constants: OS=linux, ARCH=x86_64, WS=gtk, NL=en_US
|
||||
Framework arguments: -product org.eclipse.epp.package.java.product
|
||||
Command-line arguments: -os linux -ws gtk -arch x86_64 -product org.eclipse.epp.package.java.product
|
||||
|
||||
!ENTRY ch.qos.logback.classic 1 0 2026-10-04 17:06:12.147
|
||||
!MESSAGE Activated before the state location was initialized. Retry after the state location is initialized.
|
||||
|
||||
!ENTRY org.eclipse.core.resources 4 567 2026-10-04 17:08:06.208
|
||||
!MESSAGE Workspace restored, but some problems occurred.
|
||||
!SUBENTRY 1 org.eclipse.core.resources 4 567 2026-10-04 17:08:06.208
|
||||
!MESSAGE Could not read metadata for '.org.eclipse.egit.core.cmp'.
|
||||
!STACK 1
|
||||
org.eclipse.core.internal.resources.ResourceException(/.org.eclipse.egit.core.cmp)[567]: java.lang.Exception: The project description file (.project) for '.org.eclipse.egit.core.cmp' is missing. This file contains important information about the project. The project will not function properly until this file is restored.
|
||||
at org.eclipse.core.internal.resources.ResourceException.provideStackTrace(ResourceException.java:42)
|
||||
at org.eclipse.core.internal.resources.ResourceException.<init>(ResourceException.java:38)
|
||||
at org.eclipse.core.internal.localstore.FileSystemResourceManager.read(FileSystemResourceManager.java:1023)
|
||||
at org.eclipse.core.internal.resources.SaveManager.restoreMetaInfo(SaveManager.java:1004)
|
||||
at org.eclipse.core.internal.resources.SaveManager.restoreMetaInfo(SaveManager.java:983)
|
||||
at org.eclipse.core.internal.resources.SaveManager.restore(SaveManager.java:825)
|
||||
at org.eclipse.core.internal.resources.SaveManager.startup(SaveManager.java:1689)
|
||||
at org.eclipse.core.internal.resources.Workspace.startup(Workspace.java:2677)
|
||||
at org.eclipse.core.internal.resources.Workspace.open(Workspace.java:2369)
|
||||
at org.eclipse.core.resources.ResourcesPlugin$WorkspaceInitCustomizer.addingService(ResourcesPlugin.java:591)
|
||||
at org.eclipse.core.resources.ResourcesPlugin$WorkspaceInitCustomizer.addingService(ResourcesPlugin.java:1)
|
||||
at org.osgi.util.tracker.ServiceTracker$Tracked.customizerAdding(ServiceTracker.java:947)
|
||||
at org.osgi.util.tracker.ServiceTracker$Tracked.customizerAdding(ServiceTracker.java:1)
|
||||
at org.osgi.util.tracker.AbstractTracked.trackAdding(AbstractTracked.java:257)
|
||||
at org.osgi.util.tracker.AbstractTracked.trackInitial(AbstractTracked.java:184)
|
||||
at org.osgi.util.tracker.ServiceTracker.open(ServiceTracker.java:324)
|
||||
at org.osgi.util.tracker.ServiceTracker.open(ServiceTracker.java:267)
|
||||
at org.eclipse.core.resources.ResourcesPlugin.start(ResourcesPlugin.java:565)
|
||||
at org.eclipse.osgi.internal.framework.BundleContextImpl$2.run(BundleContextImpl.java:838)
|
||||
at org.eclipse.osgi.internal.framework.BundleContextImpl$2.run(BundleContextImpl.java:1)
|
||||
at java.base/java.security.AccessController.doPrivileged(AccessController.java:251)
|
||||
at org.eclipse.osgi.internal.framework.BundleContextImpl.startActivator(BundleContextImpl.java:830)
|
||||
at org.eclipse.osgi.internal.framework.BundleContextImpl.start(BundleContextImpl.java:780)
|
||||
at org.eclipse.osgi.internal.framework.EquinoxBundle.startWorker0(EquinoxBundle.java:1071)
|
||||
at org.eclipse.osgi.internal.framework.EquinoxBundle$EquinoxModule.startWorker(EquinoxBundle.java:394)
|
||||
at org.eclipse.osgi.container.Module.doStart(Module.java:643)
|
||||
at org.eclipse.osgi.container.Module.start(Module.java:500)
|
||||
at org.eclipse.osgi.framework.util.SecureAction.start(SecureAction.java:528)
|
||||
at org.eclipse.osgi.internal.hooks.EclipseLazyStarter.postFindLocalClass(EclipseLazyStarter.java:122)
|
||||
at org.eclipse.osgi.internal.loader.classpath.ClasspathManager.findLocalClass(ClasspathManager.java:620)
|
||||
at org.eclipse.osgi.internal.loader.ModuleClassLoader.findLocalClass(ModuleClassLoader.java:353)
|
||||
at org.eclipse.osgi.internal.loader.BundleLoader.findLocalClass(BundleLoader.java:418)
|
||||
at org.eclipse.osgi.internal.loader.sources.SingleSourcePackage.loadClass(SingleSourcePackage.java:41)
|
||||
at org.eclipse.osgi.internal.loader.BundleLoader.findClass0(BundleLoader.java:523)
|
||||
at org.eclipse.osgi.internal.loader.BundleLoader.findClass(BundleLoader.java:439)
|
||||
at org.eclipse.osgi.internal.loader.ModuleClassLoader.loadClass(ModuleClassLoader.java:195)
|
||||
at java.base/java.lang.ClassLoader.loadClass(ClassLoader.java:490)
|
||||
at org.eclipse.ui.internal.ide.application.IDEApplication.start(IDEApplication.java:194)
|
||||
at org.eclipse.equinox.internal.app.EclipseAppHandle.run(EclipseAppHandle.java:219)
|
||||
at org.eclipse.core.runtime.internal.adaptor.EclipseAppLauncher.runApplication(EclipseAppLauncher.java:149)
|
||||
at org.eclipse.core.runtime.internal.adaptor.EclipseAppLauncher.start(EclipseAppLauncher.java:115)
|
||||
at org.eclipse.core.runtime.adaptor.EclipseStarter.run(EclipseStarter.java:467)
|
||||
at org.eclipse.core.runtime.adaptor.EclipseStarter.run(EclipseStarter.java:298)
|
||||
at java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:104)
|
||||
at java.base/java.lang.reflect.Method.invoke(Method.java:565)
|
||||
at org.eclipse.equinox.launcher.Main.invokeFramework(Main.java:615)
|
||||
at org.eclipse.equinox.launcher.Main.basicRun(Main.java:563)
|
||||
at org.eclipse.equinox.launcher.Main.run(Main.java:1415)
|
||||
at org.eclipse.equinox.launcher.Main.main(Main.java:1387)
|
||||
!SUBENTRY 2 org.eclipse.core.resources 4 567 2026-10-04 17:08:06.210
|
||||
!MESSAGE The project description file (.project) for '.org.eclipse.egit.core.cmp' is missing. This file contains important information about the project. The project will not function properly until this file is restored.
|
||||
!STACK 0
|
||||
java.lang.Exception: The project description file (.project) for '.org.eclipse.egit.core.cmp' is missing. This file contains important information about the project. The project will not function properly until this file is restored.
|
||||
at org.eclipse.core.internal.resources.ResourceException.provideStackTrace(ResourceException.java:42)
|
||||
at org.eclipse.core.internal.resources.ResourceException.<init>(ResourceException.java:38)
|
||||
at org.eclipse.core.internal.localstore.FileSystemResourceManager.read(FileSystemResourceManager.java:1023)
|
||||
at org.eclipse.core.internal.resources.SaveManager.restoreMetaInfo(SaveManager.java:1004)
|
||||
at org.eclipse.core.internal.resources.SaveManager.restoreMetaInfo(SaveManager.java:983)
|
||||
at org.eclipse.core.internal.resources.SaveManager.restore(SaveManager.java:825)
|
||||
at org.eclipse.core.internal.resources.SaveManager.startup(SaveManager.java:1689)
|
||||
at org.eclipse.core.internal.resources.Workspace.startup(Workspace.java:2677)
|
||||
at org.eclipse.core.internal.resources.Workspace.open(Workspace.java:2369)
|
||||
at org.eclipse.core.resources.ResourcesPlugin$WorkspaceInitCustomizer.addingService(ResourcesPlugin.java:591)
|
||||
at org.eclipse.core.resources.ResourcesPlugin$WorkspaceInitCustomizer.addingService(ResourcesPlugin.java:1)
|
||||
at org.osgi.util.tracker.ServiceTracker$Tracked.customizerAdding(ServiceTracker.java:947)
|
||||
at org.osgi.util.tracker.ServiceTracker$Tracked.customizerAdding(ServiceTracker.java:1)
|
||||
at org.osgi.util.tracker.AbstractTracked.trackAdding(AbstractTracked.java:257)
|
||||
at org.osgi.util.tracker.AbstractTracked.trackInitial(AbstractTracked.java:184)
|
||||
at org.osgi.util.tracker.ServiceTracker.open(ServiceTracker.java:324)
|
||||
at org.osgi.util.tracker.ServiceTracker.open(ServiceTracker.java:267)
|
||||
at org.eclipse.core.resources.ResourcesPlugin.start(ResourcesPlugin.java:565)
|
||||
at org.eclipse.osgi.internal.framework.BundleContextImpl$2.run(BundleContextImpl.java:838)
|
||||
at org.eclipse.osgi.internal.framework.BundleContextImpl$2.run(BundleContextImpl.java:1)
|
||||
at java.base/java.security.AccessController.doPrivileged(AccessController.java:251)
|
||||
at org.eclipse.osgi.internal.framework.BundleContextImpl.startActivator(BundleContextImpl.java:830)
|
||||
at org.eclipse.osgi.internal.framework.BundleContextImpl.start(BundleContextImpl.java:780)
|
||||
at org.eclipse.osgi.internal.framework.EquinoxBundle.startWorker0(EquinoxBundle.java:1071)
|
||||
at org.eclipse.osgi.internal.framework.EquinoxBundle$EquinoxModule.startWorker(EquinoxBundle.java:394)
|
||||
at org.eclipse.osgi.container.Module.doStart(Module.java:643)
|
||||
at org.eclipse.osgi.container.Module.start(Module.java:500)
|
||||
at org.eclipse.osgi.framework.util.SecureAction.start(SecureAction.java:528)
|
||||
at org.eclipse.osgi.internal.hooks.EclipseLazyStarter.postFindLocalClass(EclipseLazyStarter.java:122)
|
||||
at org.eclipse.osgi.internal.loader.classpath.ClasspathManager.findLocalClass(ClasspathManager.java:620)
|
||||
at org.eclipse.osgi.internal.loader.ModuleClassLoader.findLocalClass(ModuleClassLoader.java:353)
|
||||
at org.eclipse.osgi.internal.loader.BundleLoader.findLocalClass(BundleLoader.java:418)
|
||||
at org.eclipse.osgi.internal.loader.sources.SingleSourcePackage.loadClass(SingleSourcePackage.java:41)
|
||||
at org.eclipse.osgi.internal.loader.BundleLoader.findClass0(BundleLoader.java:523)
|
||||
at org.eclipse.osgi.internal.loader.BundleLoader.findClass(BundleLoader.java:439)
|
||||
at org.eclipse.osgi.internal.loader.ModuleClassLoader.loadClass(ModuleClassLoader.java:195)
|
||||
at java.base/java.lang.ClassLoader.loadClass(ClassLoader.java:490)
|
||||
at org.eclipse.ui.internal.ide.application.IDEApplication.start(IDEApplication.java:194)
|
||||
at org.eclipse.equinox.internal.app.EclipseAppHandle.run(EclipseAppHandle.java:219)
|
||||
at org.eclipse.core.runtime.internal.adaptor.EclipseAppLauncher.runApplication(EclipseAppLauncher.java:149)
|
||||
at org.eclipse.core.runtime.internal.adaptor.EclipseAppLauncher.start(EclipseAppLauncher.java:115)
|
||||
at org.eclipse.core.runtime.adaptor.EclipseStarter.run(EclipseStarter.java:467)
|
||||
at org.eclipse.core.runtime.adaptor.EclipseStarter.run(EclipseStarter.java:298)
|
||||
at java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:104)
|
||||
at java.base/java.lang.reflect.Method.invoke(Method.java:565)
|
||||
at org.eclipse.equinox.launcher.Main.invokeFramework(Main.java:615)
|
||||
at org.eclipse.equinox.launcher.Main.basicRun(Main.java:563)
|
||||
at org.eclipse.equinox.launcher.Main.run(Main.java:1415)
|
||||
at org.eclipse.equinox.launcher.Main.main(Main.java:1387)
|
||||
|
||||
!ENTRY ch.qos.logback.classic 1 0 2026-10-04 17:08:06.460
|
||||
!MESSAGE Logback config file: /home/Kotama_Chio/Musique/SillyBlahaj.org/Java/.metadata/.plugins/org.eclipse.m2e.logback/logback.2.7.201.20260831-2044.xml
|
||||
!SESSION 2026-10-04 22:23:01.219 -----------------------------------------------
|
||||
eclipse.buildId=4.41.0.20260903-0719
|
||||
java.version=25.0.4.1
|
||||
java.vendor=Eclipse Adoptium
|
||||
BootLoader constants: OS=linux, ARCH=x86_64, WS=gtk, NL=en_US
|
||||
Framework arguments: -product org.eclipse.epp.package.java.product
|
||||
Command-line arguments: -os linux -ws gtk -arch x86_64 -product org.eclipse.epp.package.java.product
|
||||
|
||||
!ENTRY ch.qos.logback.classic 1 0 2026-10-04 22:23:04.077
|
||||
!MESSAGE Activated before the state location was initialized. Retry after the state location is initialized.
|
||||
|
||||
!ENTRY ch.qos.logback.classic 1 0 2026-10-04 22:27:28.053
|
||||
!MESSAGE Logback config file: /home/Kotama_Chio/Musique/SillyBlahaj.org/Java/.metadata/.plugins/org.eclipse.m2e.logback/logback.2.7.201.20260831-2044.xml
|
||||
!SESSION 2026-10-06 13:16:14.328 -----------------------------------------------
|
||||
eclipse.buildId=4.41.0.20260903-0719
|
||||
java.version=25.0.4.1
|
||||
java.vendor=Eclipse Adoptium
|
||||
BootLoader constants: OS=linux, ARCH=x86_64, WS=gtk, NL=en_US
|
||||
Framework arguments: -product org.eclipse.epp.package.java.product
|
||||
Command-line arguments: -os linux -ws gtk -arch x86_64 -product org.eclipse.epp.package.java.product
|
||||
|
||||
!ENTRY ch.qos.logback.classic 1 0 2026-10-06 13:16:16.870
|
||||
!MESSAGE Activated before the state location was initialized. Retry after the state location is initialized.
|
||||
|
||||
!ENTRY ch.qos.logback.classic 1 0 2026-10-06 13:16:26.800
|
||||
!MESSAGE Logback config file: /home/Kotama_Chio/Musique/SillyBlahaj.org/Java/.metadata/.plugins/org.eclipse.m2e.logback/logback.2.7.201.20260831-2044.xml
|
||||
!SESSION 2026-10-07 00:08:01.488 -----------------------------------------------
|
||||
eclipse.buildId=4.41.0.20260903-0719
|
||||
java.version=25.0.4.1
|
||||
java.vendor=Eclipse Adoptium
|
||||
BootLoader constants: OS=linux, ARCH=x86_64, WS=gtk, NL=en_US
|
||||
Framework arguments: -product org.eclipse.epp.package.java.product
|
||||
Command-line arguments: -os linux -ws gtk -arch x86_64 -product org.eclipse.epp.package.java.product
|
||||
|
||||
!ENTRY ch.qos.logback.classic 1 0 2026-10-07 00:08:05.027
|
||||
!MESSAGE Activated before the state location was initialized. Retry after the state location is initialized.
|
||||
|
||||
!ENTRY ch.qos.logback.classic 1 0 2026-10-07 00:08:11.894
|
||||
!MESSAGE Logback config file: /home/Kotama_Chio/Musique/SillyBlahaj.org/Java/.metadata/.plugins/org.eclipse.m2e.logback/logback.2.7.201.20260831-2044.xml
|
||||
|
||||
!ENTRY org.eclipse.e4.ui.workbench 4 0 2026-10-07 00:08:12.044
|
||||
!MESSAGE Unable to load resource file:/home/Kotama_Chio/Musique/SillyBlahaj.org/Java/.metadata/.plugins/org.eclipse.e4.workbench/workbench.xmi
|
||||
!STACK 0
|
||||
org.eclipse.emf.ecore.resource.impl.ResourceSetImpl$1DiagnosticWrappedException: org.xml.sax.SAXParseExceptionpublicId: file:/home/Kotama_Chio/Musique/SillyBlahaj.org/Java/.metadata/.plugins/org.eclipse.e4.workbench/workbench.xmi; systemId: file:/home/Kotama_Chio/Musique/SillyBlahaj.org/Java/.metadata/.plugins/org.eclipse.e4.workbench/workbench.xmi; lineNumber: 1; columnNumber: 1; Premature end of file.
|
||||
at org.eclipse.emf.ecore.resource.impl.ResourceSetImpl.handleDemandLoadException(ResourceSetImpl.java:319)
|
||||
at org.eclipse.emf.ecore.resource.impl.ResourceSetImpl.demandLoadHelper(ResourceSetImpl.java:278)
|
||||
at org.eclipse.emf.ecore.resource.impl.ResourceSetImpl.getResource(ResourceSetImpl.java:406)
|
||||
at org.eclipse.e4.ui.internal.workbench.ResourceHandler.getResource(ResourceHandler.java:287)
|
||||
at org.eclipse.e4.ui.internal.workbench.ResourceHandler.loadResource(ResourceHandler.java:263)
|
||||
at org.eclipse.e4.ui.internal.workbench.ResourceHandler.loadMostRecentModel(ResourceHandler.java:166)
|
||||
at org.eclipse.e4.ui.internal.workbench.swt.E4Application.loadApplicationModel(E4Application.java:394)
|
||||
at org.eclipse.e4.ui.internal.workbench.swt.E4Application.createE4Workbench(E4Application.java:255)
|
||||
at org.eclipse.ui.internal.Workbench.lambda$3(Workbench.java:599)
|
||||
at org.eclipse.core.databinding.observable.Realm.runWithDefault(Realm.java:339)
|
||||
at org.eclipse.ui.internal.Workbench.createAndRunWorkbench(Workbench.java:583)
|
||||
at org.eclipse.ui.PlatformUI.createAndRunWorkbench(PlatformUI.java:173)
|
||||
at org.eclipse.ui.internal.ide.application.IDEApplication.start(IDEApplication.java:193)
|
||||
at org.eclipse.equinox.internal.app.EclipseAppHandle.run(EclipseAppHandle.java:219)
|
||||
at org.eclipse.core.runtime.internal.adaptor.EclipseAppLauncher.runApplication(EclipseAppLauncher.java:149)
|
||||
at org.eclipse.core.runtime.internal.adaptor.EclipseAppLauncher.start(EclipseAppLauncher.java:115)
|
||||
at org.eclipse.core.runtime.adaptor.EclipseStarter.run(EclipseStarter.java:467)
|
||||
at org.eclipse.core.runtime.adaptor.EclipseStarter.run(EclipseStarter.java:298)
|
||||
at java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:104)
|
||||
at java.base/java.lang.reflect.Method.invoke(Method.java:565)
|
||||
at org.eclipse.equinox.launcher.Main.invokeFramework(Main.java:615)
|
||||
at org.eclipse.equinox.launcher.Main.basicRun(Main.java:563)
|
||||
at org.eclipse.equinox.launcher.Main.run(Main.java:1415)
|
||||
at org.eclipse.equinox.launcher.Main.main(Main.java:1387)
|
||||
Caused by: org.xml.sax.SAXParseExceptionpublicId: file:/home/Kotama_Chio/Musique/SillyBlahaj.org/Java/.metadata/.plugins/org.eclipse.e4.workbench/workbench.xmi; systemId: file:/home/Kotama_Chio/Musique/SillyBlahaj.org/Java/.metadata/.plugins/org.eclipse.e4.workbench/workbench.xmi; lineNumber: 1; columnNumber: 1; Premature end of file.
|
||||
at java.xml/com.sun.org.apache.xerces.internal.util.ErrorHandlerWrapper.createSAXParseException(ErrorHandlerWrapper.java:204)
|
||||
at java.xml/com.sun.org.apache.xerces.internal.util.ErrorHandlerWrapper.fatalError(ErrorHandlerWrapper.java:178)
|
||||
at java.xml/com.sun.org.apache.xerces.internal.impl.XMLErrorReporter.reportError(XMLErrorReporter.java:400)
|
||||
at java.xml/com.sun.org.apache.xerces.internal.impl.XMLErrorReporter.reportError(XMLErrorReporter.java:327)
|
||||
at java.xml/com.sun.org.apache.xerces.internal.impl.XMLScanner.reportFatalError(XMLScanner.java:1465)
|
||||
at java.xml/com.sun.org.apache.xerces.internal.impl.XMLDocumentScannerImpl$PrologDriver.next(XMLDocumentScannerImpl.java:1043)
|
||||
at java.xml/com.sun.org.apache.xerces.internal.impl.XMLDocumentScannerImpl.next(XMLDocumentScannerImpl.java:635)
|
||||
at java.xml/com.sun.org.apache.xerces.internal.impl.XMLNSDocumentScannerImpl.next(XMLNSDocumentScannerImpl.java:113)
|
||||
at java.xml/com.sun.org.apache.xerces.internal.impl.XMLDocumentFragmentScannerImpl.scanDocument(XMLDocumentFragmentScannerImpl.java:551)
|
||||
at java.xml/com.sun.org.apache.xerces.internal.parsers.XML11Configuration.parse(XML11Configuration.java:890)
|
||||
at java.xml/com.sun.org.apache.xerces.internal.parsers.XML11Configuration.parse(XML11Configuration.java:826)
|
||||
at java.xml/com.sun.org.apache.xerces.internal.parsers.XMLParser.parse(XMLParser.java:137)
|
||||
at java.xml/com.sun.org.apache.xerces.internal.parsers.AbstractSAXParser.parse(AbstractSAXParser.java:1225)
|
||||
at java.xml/com.sun.org.apache.xerces.internal.jaxp.SAXParserImpl$JAXPSAXParser.parse(SAXParserImpl.java:629)
|
||||
at java.xml/com.sun.org.apache.xerces.internal.jaxp.SAXParserImpl.parse(SAXParserImpl.java:328)
|
||||
at org.eclipse.emf.ecore.xmi.impl.XMLLoadImpl.load(XMLLoadImpl.java:175)
|
||||
at org.eclipse.emf.ecore.xmi.impl.XMLResourceImpl.doLoad(XMLResourceImpl.java:260)
|
||||
at org.eclipse.emf.ecore.resource.impl.ResourceImpl.load(ResourceImpl.java:1563)
|
||||
at org.eclipse.emf.ecore.resource.impl.ResourceImpl.load(ResourceImpl.java:1342)
|
||||
at org.eclipse.emf.ecore.resource.impl.ResourceSetImpl.demandLoad(ResourceSetImpl.java:259)
|
||||
at org.eclipse.emf.ecore.resource.impl.ResourceSetImpl.demandLoadHelper(ResourceSetImpl.java:274)
|
||||
... 22 more
|
||||
|
||||
!ENTRY org.eclipse.e4.ui.workbench 4 0 2026-10-07 00:08:12.045
|
||||
!MESSAGE The persisted application model has no top-level window. Reinitializing with the default application model.
|
||||
!STACK 0
|
||||
java.lang.Exception
|
||||
at org.eclipse.e4.ui.internal.workbench.ResourceHandler.loadMostRecentModel(ResourceHandler.java:171)
|
||||
at org.eclipse.e4.ui.internal.workbench.swt.E4Application.loadApplicationModel(E4Application.java:394)
|
||||
at org.eclipse.e4.ui.internal.workbench.swt.E4Application.createE4Workbench(E4Application.java:255)
|
||||
at org.eclipse.ui.internal.Workbench.lambda$3(Workbench.java:599)
|
||||
at org.eclipse.core.databinding.observable.Realm.runWithDefault(Realm.java:339)
|
||||
at org.eclipse.ui.internal.Workbench.createAndRunWorkbench(Workbench.java:583)
|
||||
at org.eclipse.ui.PlatformUI.createAndRunWorkbench(PlatformUI.java:173)
|
||||
at org.eclipse.ui.internal.ide.application.IDEApplication.start(IDEApplication.java:193)
|
||||
at org.eclipse.equinox.internal.app.EclipseAppHandle.run(EclipseAppHandle.java:219)
|
||||
at org.eclipse.core.runtime.internal.adaptor.EclipseAppLauncher.runApplication(EclipseAppLauncher.java:149)
|
||||
at org.eclipse.core.runtime.internal.adaptor.EclipseAppLauncher.start(EclipseAppLauncher.java:115)
|
||||
at org.eclipse.core.runtime.adaptor.EclipseStarter.run(EclipseStarter.java:467)
|
||||
at org.eclipse.core.runtime.adaptor.EclipseStarter.run(EclipseStarter.java:298)
|
||||
at java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:104)
|
||||
at java.base/java.lang.reflect.Method.invoke(Method.java:565)
|
||||
at org.eclipse.equinox.launcher.Main.invokeFramework(Main.java:615)
|
||||
at org.eclipse.equinox.launcher.Main.basicRun(Main.java:563)
|
||||
at org.eclipse.equinox.launcher.Main.run(Main.java:1415)
|
||||
at org.eclipse.equinox.launcher.Main.main(Main.java:1387)
|
||||
!SESSION 2026-10-07 16:37:19.753 -----------------------------------------------
|
||||
eclipse.buildId=4.41.0.20260903-0719
|
||||
java.version=25.0.4.1
|
||||
java.vendor=Eclipse Adoptium
|
||||
BootLoader constants: OS=linux, ARCH=x86_64, WS=gtk, NL=en_US
|
||||
Framework arguments: -product org.eclipse.epp.package.java.product
|
||||
Command-line arguments: -os linux -ws gtk -arch x86_64 -product org.eclipse.epp.package.java.product
|
||||
|
||||
!ENTRY ch.qos.logback.classic 1 0 2026-10-07 16:37:21.324
|
||||
!MESSAGE Activated before the state location was initialized. Retry after the state location is initialized.
|
||||
|
||||
!ENTRY ch.qos.logback.classic 1 0 2026-10-07 16:37:34.953
|
||||
!MESSAGE Logback config file: /home/Kotama_Chio/Musique/SillyBlahaj.org/Java/.metadata/.plugins/org.eclipse.m2e.logback/logback.2.7.201.20260831-2044.xml
|
||||
!SESSION 2026-10-07 21:49:54.200 -----------------------------------------------
|
||||
eclipse.buildId=4.41.0.20260903-0719
|
||||
java.version=25.0.4.1
|
||||
java.vendor=Eclipse Adoptium
|
||||
BootLoader constants: OS=linux, ARCH=x86_64, WS=gtk, NL=en_US
|
||||
Framework arguments: -product org.eclipse.epp.package.java.product
|
||||
Command-line arguments: -os linux -ws gtk -arch x86_64 -product org.eclipse.epp.package.java.product
|
||||
|
||||
!ENTRY ch.qos.logback.classic 1 0 2026-10-07 21:49:56.947
|
||||
!MESSAGE Activated before the state location was initialized. Retry after the state location is initialized.
|
||||
|
||||
!ENTRY ch.qos.logback.classic 1 0 2026-10-07 21:50:00.010
|
||||
!MESSAGE Logback config file: /home/Kotama_Chio/Musique/SillyBlahaj.org/Java/.metadata/.plugins/org.eclipse.m2e.logback/logback.2.7.201.20260831-2044.xml
|
||||
|
||||
!ENTRY org.eclipse.oomph.ui 4 0 2026-10-07 21:50:13.676
|
||||
!MESSAGE java.lang.ExceptionInInitializerError
|
||||
!STACK 0
|
||||
java.lang.ExceptionInInitializerError
|
||||
at org.eclipse.oomph.setup.ui.recorder.RecorderManager.<clinit>(RecorderManager.java:114)
|
||||
at org.eclipse.oomph.setup.ui.SetupUIPlugin.performStartup(SetupUIPlugin.java:230)
|
||||
at org.eclipse.oomph.setup.ui.EarlyStartup.earlyStartup(EarlyStartup.java:23)
|
||||
at org.eclipse.oomph.internal.ui.EarlyStart.run(EarlyStart.java:119)
|
||||
at org.eclipse.oomph.internal.ui.EarlyStart$1.run(EarlyStart.java:98)
|
||||
at org.eclipse.core.internal.jobs.Worker.run(Worker.java:63)
|
||||
Caused by: java.lang.NullPointerException: Cannot invoke "org.osgi.framework.BundleContext.getServiceReference(java.lang.Class)" because "context" is null
|
||||
at org.eclipse.oomph.p2.internal.core.AgentManagerImpl.getCurrentAgent(AgentManagerImpl.java:207)
|
||||
at org.eclipse.oomph.p2.core.P2Util.getCurrentProvisioningAgent(P2Util.java:117)
|
||||
at org.eclipse.oomph.setup.internal.core.util.SetupCoreUtil.<clinit>(SetupCoreUtil.java:158)
|
||||
... 6 more
|
||||
!SESSION 2026-10-09 18:49:48.691 -----------------------------------------------
|
||||
eclipse.buildId=4.41.0.20260903-0719
|
||||
java.version=25.0.4.1
|
||||
java.vendor=Eclipse Adoptium
|
||||
BootLoader constants: OS=linux, ARCH=x86_64, WS=gtk, NL=en_US
|
||||
Framework arguments: -product org.eclipse.epp.package.java.product
|
||||
Command-line arguments: -os linux -ws gtk -arch x86_64 -product org.eclipse.epp.package.java.product
|
||||
|
||||
!ENTRY ch.qos.logback.classic 1 0 2026-10-09 18:49:50.177
|
||||
!MESSAGE Activated before the state location was initialized. Retry after the state location is initialized.
|
||||
|
||||
!ENTRY ch.qos.logback.classic 1 0 2026-10-09 18:50:02.098
|
||||
!MESSAGE Logback config file: /home/Kotama_Chio/Musique/SillyBlahaj.org/Java/.metadata/.plugins/org.eclipse.m2e.logback/logback.2.7.201.20260831-2044.xml
|
||||
!SESSION 2026-10-10 23:02:21.698 -----------------------------------------------
|
||||
eclipse.buildId=4.41.0.20260903-0719
|
||||
java.version=25.0.4.1
|
||||
java.vendor=Eclipse Adoptium
|
||||
BootLoader constants: OS=linux, ARCH=x86_64, WS=gtk, NL=en_US
|
||||
Framework arguments: -product org.eclipse.epp.package.java.product
|
||||
Command-line arguments: -os linux -ws gtk -arch x86_64 -product org.eclipse.epp.package.java.product
|
||||
|
||||
!ENTRY ch.qos.logback.classic 1 0 2026-10-10 23:02:24.629
|
||||
!MESSAGE Activated before the state location was initialized. Retry after the state location is initialized.
|
||||
|
||||
!ENTRY ch.qos.logback.classic 1 0 2026-10-10 23:02:27.597
|
||||
!MESSAGE Logback config file: /home/Kotama_Chio/Musique/SillyBlahaj.org/Java/.metadata/.plugins/org.eclipse.m2e.logback/logback.2.7.201.20260831-2044.xml
|
||||
+150
@@ -0,0 +1,150 @@
|
||||
package org.sillyjirafe.db;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.DriverManager;
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
|
||||
public class Crawl_StateDB
|
||||
{
|
||||
String URL = new String();
|
||||
String insertSql = "INSERT OR IGNORE INTO crawlstates (bind_name, last_page, rnd, error) VALUES (?, ?, ?, ?)";
|
||||
Connection connection;
|
||||
|
||||
public Crawl_StateDB(String BDstr) throws SQLException
|
||||
{
|
||||
URL = BDstr;
|
||||
connection = DriverManager.getConnection(URL);
|
||||
}
|
||||
|
||||
public void LinkToDB() throws SQLException
|
||||
{
|
||||
try (Statement stmt = connection.createStatement())
|
||||
{
|
||||
stmt.execute("""
|
||||
CREATE TABLE IF NOT EXISTS crawlstates (
|
||||
bind_name TEXT PRIMARY KEY,
|
||||
last_page INTEGER NOT NULL DEFAULT 1,
|
||||
rnd TEXT NOT NULL DEFAULT 'a',
|
||||
error INTEGER NOT NULL DEFAULT 0,
|
||||
)
|
||||
""");
|
||||
}
|
||||
}
|
||||
|
||||
public void AddEllement(String bind, int lastpage) throws SQLException
|
||||
{
|
||||
try (PreparedStatement ps = connection.prepareStatement(insertSql))
|
||||
{
|
||||
ps.setString(1, bind);
|
||||
ps.setInt(2, lastpage);
|
||||
ps.executeUpdate();
|
||||
System.out.printf("%s is now page %d \n", bind, lastpage);
|
||||
}
|
||||
}
|
||||
|
||||
public int getPage(String bindName) throws SQLException {
|
||||
String sql = "SELECT last_page FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("last_page");
|
||||
} else {
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
public String getRND(String bindName) throws SQLException {
|
||||
String sql = "SELECT rnd FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getString("rnd");
|
||||
} else {
|
||||
return "a";
|
||||
}
|
||||
}
|
||||
}
|
||||
public int getErrors(String bindName) throws SQLException {
|
||||
String sql = "SELECT error FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("error");
|
||||
} else {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
public void updatePage(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET last_page = last_page + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void newError(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET error = error + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void resetError(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET error = 0
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void resetPage(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET last_page = 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void updateletter(String bindName) throws SQLException
|
||||
{
|
||||
if (getErrors(bindName) >= 10)
|
||||
{
|
||||
resetError(bindName);
|
||||
resetPage(bindName);
|
||||
}
|
||||
}
|
||||
|
||||
public void Close() throws SQLException
|
||||
{
|
||||
if (connection != null)
|
||||
{
|
||||
connection.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
+176
@@ -0,0 +1,176 @@
|
||||
package org.sillyjirafe.db;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.DriverManager;
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
|
||||
public class Crawl_StateDB
|
||||
{
|
||||
String URL = new String();
|
||||
String insertSql = "INSERT OR IGNORE INTO crawlstates (bind_name, last_page, rnd, error) VALUES (?, ?, ?, ?)";
|
||||
Connection connection;
|
||||
|
||||
public Crawl_StateDB(String BDstr) throws SQLException
|
||||
{
|
||||
URL = BDstr;
|
||||
connection = DriverManager.getConnection(URL);
|
||||
}
|
||||
|
||||
public void LinkToDB() throws SQLException
|
||||
{
|
||||
try (Statement stmt = connection.createStatement())
|
||||
{
|
||||
stmt.execute("""
|
||||
CREATE TABLE IF NOT EXISTS crawlstates (
|
||||
bind_name TEXT PRIMARY KEY,
|
||||
last_page INTEGER NOT NULL DEFAULT 1,
|
||||
rnd TEXT NOT NULL DEFAULT 'a',
|
||||
error INTEGER NOT NULL DEFAULT 0,
|
||||
)
|
||||
""");
|
||||
}
|
||||
}
|
||||
|
||||
public void AddEllement(String bind, int lastpage) throws SQLException
|
||||
{
|
||||
try (PreparedStatement ps = connection.prepareStatement(insertSql))
|
||||
{
|
||||
ps.setString(1, bind);
|
||||
ps.setInt(2, lastpage);
|
||||
ps.executeUpdate();
|
||||
System.out.printf("%s is now page %d \n", bind, lastpage);
|
||||
}
|
||||
}
|
||||
|
||||
public int getPage(String bindName) throws SQLException {
|
||||
String sql = "SELECT last_page FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("last_page");
|
||||
} else {
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
public String getRND(String bindName) throws SQLException {
|
||||
String sql = "SELECT rnd FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getString("rnd");
|
||||
} else {
|
||||
return "a";
|
||||
}
|
||||
}
|
||||
}
|
||||
public int getErrors(String bindName) throws SQLException {
|
||||
String sql = "SELECT error FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("error");
|
||||
} else {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
public void updatePage(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET last_page = last_page + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void setRnd(String bindName, String c) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET last_page = (rnd)
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void newError(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET error = error + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void resetError(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET error = 0
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void resetPage(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET last_page = 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public static String nextLetter(String current) {
|
||||
char[] lettres = current.toCharArray();
|
||||
int i = lettres.length - 1;
|
||||
|
||||
while (i >= 0) {
|
||||
if (lettres[i] < 'z') {
|
||||
lettres[i]++;
|
||||
return new String(lettres);
|
||||
}
|
||||
lettres[i] = 'a';
|
||||
i--;
|
||||
}
|
||||
return "a" + new String(lettres);
|
||||
}
|
||||
public void updateletter(String bindName) throws SQLException
|
||||
{
|
||||
if (getErrors(bindName) >= 10)
|
||||
{
|
||||
setRnd(bindName, nextLetter(getRND(bindName)));
|
||||
resetError(bindName);
|
||||
resetPage(bindName);
|
||||
}
|
||||
}
|
||||
|
||||
public void Close() throws SQLException
|
||||
{
|
||||
if (connection != null)
|
||||
{
|
||||
connection.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
+44
@@ -0,0 +1,44 @@
|
||||
package org.sillyjirafe.uoapi.binds;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.json.JSONArray;
|
||||
import org.json.JSONObject;
|
||||
import org.sillyjirafe.uoapi.Api;
|
||||
import org.sillyjirafe.uoapi.element;
|
||||
|
||||
public class PixabayImagesApi implements Api
|
||||
{
|
||||
public String getName()
|
||||
{
|
||||
return "PixabayImagesApi";
|
||||
}
|
||||
|
||||
public String getURL(int page, String apikey, String rnd) {
|
||||
return "https://pixabay.com/api/?per_page=200&page=" + page + "&key=" + apikey;
|
||||
}
|
||||
|
||||
public List<element> Parse(String jsonRaw) {
|
||||
List<element> resultats = new ArrayList<>();
|
||||
|
||||
try {
|
||||
JSONObject json = new JSONObject(jsonRaw);
|
||||
JSONArray items = json.getJSONArray("hits");
|
||||
|
||||
for (int i = 0; i < items.length(); i++) {
|
||||
JSONObject repo = items.getJSONObject(i);
|
||||
|
||||
String titre = repo.getString("name");
|
||||
String url = repo.getString("pageURL");
|
||||
String licence = "cc0";
|
||||
|
||||
resultats.add(new element(titre, url, licence, "pixabay", "image"));
|
||||
}
|
||||
} catch (org.json.JSONException e) {
|
||||
System.out.println("error but continue : " + e.getMessage());
|
||||
}
|
||||
|
||||
return resultats;
|
||||
}
|
||||
}
|
||||
+151
@@ -0,0 +1,151 @@
|
||||
package org.sillyjirafe.db;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.DriverManager;
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
|
||||
public class Crawl_StateDB
|
||||
{
|
||||
String URL = new String();
|
||||
String insertSql = "INSERT OR IGNORE INTO crawlstates (bind_name, last_page, rnd, error) VALUES (?, ?, ?, ?)";
|
||||
Connection connection;
|
||||
|
||||
public Crawl_StateDB(String BDstr) throws SQLException
|
||||
{
|
||||
URL = BDstr;
|
||||
connection = DriverManager.getConnection(URL);
|
||||
}
|
||||
|
||||
public void LinkToDB() throws SQLException
|
||||
{
|
||||
try (Statement stmt = connection.createStatement())
|
||||
{
|
||||
stmt.execute("""
|
||||
CREATE TABLE IF NOT EXISTS crawlstates (
|
||||
bind_name TEXT PRIMARY KEY,
|
||||
last_page INTEGER NOT NULL DEFAULT 1,
|
||||
rnd TEXT NOT NULL DEFAULT 'a',
|
||||
error INTEGER NOT NULL DEFAULT 0,
|
||||
)
|
||||
""");
|
||||
}
|
||||
}
|
||||
|
||||
public void AddEllement(String bind, int lastpage) throws SQLException
|
||||
{
|
||||
try (PreparedStatement ps = connection.prepareStatement(insertSql))
|
||||
{
|
||||
ps.setString(1, bind);
|
||||
ps.setInt(2, lastpage);
|
||||
ps.setString(3, rnd);
|
||||
ps.executeUpdate();
|
||||
System.out.printf("%s is now page %d \n", bind, lastpage);
|
||||
}
|
||||
}
|
||||
|
||||
public int getPage(String bindName) throws SQLException {
|
||||
String sql = "SELECT last_page FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("last_page");
|
||||
} else {
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
public String getRND(String bindName) throws SQLException {
|
||||
String sql = "SELECT rnd FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getString("rnd");
|
||||
} else {
|
||||
return "a";
|
||||
}
|
||||
}
|
||||
}
|
||||
public int getErrors(String bindName) throws SQLException {
|
||||
String sql = "SELECT error FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("error");
|
||||
} else {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
public void updatePage(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET last_page = last_page + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void newError(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET error = error + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void resetError(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET error = 0
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void resetPage(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET last_page = 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void updateletter(String bindName) throws SQLException
|
||||
{
|
||||
if (getErrors(bindName) >= 10)
|
||||
{
|
||||
resetError(bindName);
|
||||
resetPage(bindName);
|
||||
}
|
||||
}
|
||||
|
||||
public void Close() throws SQLException
|
||||
{
|
||||
if (connection != null)
|
||||
{
|
||||
connection.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
+44
@@ -0,0 +1,44 @@
|
||||
package org.sillyjirafe.uoapi.binds;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.json.JSONArray;
|
||||
import org.json.JSONObject;
|
||||
import org.sillyjirafe.uoapi.Api;
|
||||
import org.sillyjirafe.uoapi.element;
|
||||
|
||||
public class GitHubApacheApi implements Api
|
||||
{
|
||||
public String getName()
|
||||
{
|
||||
return "GitHubApacheApi";
|
||||
}
|
||||
|
||||
public String getURL(int page, String apikey, String rnd) {
|
||||
return "https://api.github.com/search/repositories?q="+ rnd +"&license:apache-2.0&page=" + page;
|
||||
}
|
||||
|
||||
public List<element> Parse(String jsonRaw) {
|
||||
List<element> resultats = new ArrayList<>();
|
||||
|
||||
try {
|
||||
JSONObject json = new JSONObject(jsonRaw);
|
||||
JSONArray items = json.getJSONArray("items");
|
||||
|
||||
for (int i = 0; i < items.length(); i++) {
|
||||
JSONObject repo = items.getJSONObject(i);
|
||||
|
||||
String titre = repo.getString("full_name");
|
||||
String url = repo.getString("html_url");
|
||||
String licence = repo.getJSONObject("license").getString("spdx_id");
|
||||
|
||||
resultats.add(new element(titre, url, licence, "github", "code"));
|
||||
}
|
||||
} catch (org.json.JSONException e) {
|
||||
System.out.println("error but continue : " + e.getMessage());
|
||||
}
|
||||
|
||||
return resultats;
|
||||
}
|
||||
}
|
||||
+46
@@ -0,0 +1,46 @@
|
||||
package org.sillyjirafe.uoapi;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import org.sillyjirafe.uoapi.binds.*;
|
||||
|
||||
public class Out {
|
||||
List<Api> Apis = new ArrayList<>();
|
||||
|
||||
public List<String> PreOut()
|
||||
{
|
||||
Apis.add(new GitHubMITApi());
|
||||
Apis.add(new GitHubLGPLApi());
|
||||
Apis.add(new GitHubApacheApi());
|
||||
Apis.add(new OpenVerseImagesApi());
|
||||
Apis.add(new OpenVerseAudiosApi());
|
||||
Apis.add(new PixabayImagesApi());
|
||||
|
||||
List<String> bindnames = new ArrayList<>();
|
||||
|
||||
for (Api api : Apis)
|
||||
{
|
||||
bindnames.add(api.getName());
|
||||
}
|
||||
|
||||
return bindnames;
|
||||
}
|
||||
|
||||
public List<element> out(Map<String, Integer> bindpages, Map<String, String> ApiKeys) throws Exception
|
||||
{
|
||||
List<element> Elements = new ArrayList<>();
|
||||
|
||||
for (int i = 0; i < Apis.size(); i++)
|
||||
{
|
||||
List<element> currentapi = Caller.CallApi(Apis.get(i), bindpages.get(Apis.get(i).getName()), ApiKeys.get(Apis.get(i).getName()));
|
||||
for (int j = 0; j < currentapi.size(); j++)
|
||||
{
|
||||
Elements.add(currentapi.get(j));
|
||||
}
|
||||
}
|
||||
return Elements;
|
||||
}
|
||||
}
|
||||
+43
@@ -0,0 +1,43 @@
|
||||
package org.sillyjirafe.uoapi.binds;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.json.JSONArray;
|
||||
import org.json.JSONObject;
|
||||
import org.sillyjirafe.uoapi.Api;
|
||||
import org.sillyjirafe.uoapi.element;
|
||||
|
||||
public class OpenVerseImagesApi implements Api
|
||||
{
|
||||
public String getName()
|
||||
{
|
||||
return "OpenVerseImagesApi";
|
||||
}
|
||||
|
||||
public String getURL(int page, String apikey, String rnd) {
|
||||
return "https://api.openverse.org/v1/images/?q=\n" + rnd+"\nformat=json&page=" + page;
|
||||
}
|
||||
|
||||
public List<element> Parse(String jsonRaw) {
|
||||
List<element> resultats = new ArrayList<>();
|
||||
|
||||
try {
|
||||
JSONObject json = new JSONObject(jsonRaw);
|
||||
JSONArray items = json.getJSONArray("results");
|
||||
|
||||
for (int i = 0; i < items.length(); i++) {
|
||||
JSONObject image = items.getJSONObject(i);
|
||||
|
||||
String titre = image.getString("title");
|
||||
String url = image.getString("url");
|
||||
String licence = image.getString("license");
|
||||
|
||||
resultats.add(new element(titre, url, licence, "openverse", "image"));
|
||||
}
|
||||
} catch (org.json.JSONException e) {
|
||||
System.out.println("error but continue : " + e.getMessage());
|
||||
}
|
||||
return resultats;
|
||||
}
|
||||
}
|
||||
+41
@@ -0,0 +1,41 @@
|
||||
package org.sillyjirafe.uoapi.binds;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.json.JSONArray;
|
||||
import org.json.JSONObject;
|
||||
import org.sillyjirafe.uoapi.Api;
|
||||
import org.sillyjirafe.uoapi.element;
|
||||
|
||||
public class OpenVerseImagesApi implements Api
|
||||
{
|
||||
public String getName()
|
||||
{
|
||||
return "OpenVerseImagesApi";
|
||||
}
|
||||
|
||||
public String getURL(int page, String apikey, String rnd) {
|
||||
return "https://api.openverse.org/v1/images/?q=\"" + rnd+"\"format=json&page=" + page;
|
||||
}
|
||||
|
||||
public List<element> Parse(String jsonRaw) {
|
||||
List<element> resultats = new ArrayList<>();
|
||||
JSONObject json = new JSONObject(jsonRaw);
|
||||
JSONArray items = json.getJSONArray("results");
|
||||
|
||||
for (int i = 0; i < items.length(); i++) {
|
||||
JSONObject image = items.getJSONObject(i);
|
||||
|
||||
String titre = image.getString("title");
|
||||
String url = image.getString("url");
|
||||
String licence = image.getString("license");
|
||||
|
||||
resultats.add(new element(titre, url, licence, "openverse", "image"));
|
||||
}
|
||||
} catch (org.json.JSONException e) {
|
||||
System.out.println("error but continue : " + e.getMessage());
|
||||
}
|
||||
return resultats;
|
||||
}
|
||||
}
|
||||
+43
@@ -0,0 +1,43 @@
|
||||
package org.sillyjirafe.uoapi.binds;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.json.JSONArray;
|
||||
import org.json.JSONObject;
|
||||
import org.sillyjirafe.uoapi.Api;
|
||||
import org.sillyjirafe.uoapi.element;
|
||||
|
||||
public class OpenVerseAudiosApi implements Api
|
||||
{
|
||||
public String getName()
|
||||
{
|
||||
return "OpenVerseAudiosApi";
|
||||
}
|
||||
|
||||
public String getURL(int page, String apikey, String rnd) {
|
||||
return "https://api.openverse.org/v1/audio/?q=\""+ rnd +"\"format=json&page=" + page;
|
||||
}
|
||||
|
||||
public List<element> Parse(String jsonRaw) {
|
||||
List<element> resultats = new ArrayList<>();
|
||||
|
||||
try {
|
||||
JSONObject json = new JSONObject(jsonRaw);
|
||||
JSONArray items = json.getJSONArray("results");
|
||||
|
||||
for (int i = 0; i < items.length(); i++) {
|
||||
JSONObject audio = items.getJSONObject(i);
|
||||
|
||||
String titre = audio.getString("title");
|
||||
String url = audio.getString("url");
|
||||
String licence = audio.getString("license");
|
||||
|
||||
resultats.add(new element(titre, url, licence, "openverse", "audio"));
|
||||
}
|
||||
} catch (org.json.JSONException e) {
|
||||
System.out.println("error but continue : " + e.getMessage());
|
||||
}
|
||||
return resultats;
|
||||
}
|
||||
}
|
||||
+164
@@ -0,0 +1,164 @@
|
||||
package org.sillyjirafe.db;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.DriverManager;
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
|
||||
public class Crawl_StateDB
|
||||
{
|
||||
String URL = new String();
|
||||
String insertSql = "INSERT OR IGNORE INTO crawlstates (bind_name, last_page, rnd, error) VALUES (?, ?, ?, ?)";
|
||||
Connection connection;
|
||||
|
||||
public Crawl_StateDB(String BDstr) throws SQLException
|
||||
{
|
||||
URL = BDstr;
|
||||
connection = DriverManager.getConnection(URL);
|
||||
}
|
||||
|
||||
public void LinkToDB() throws SQLException
|
||||
{
|
||||
try (Statement stmt = connection.createStatement())
|
||||
{
|
||||
stmt.execute("""
|
||||
CREATE TABLE IF NOT EXISTS crawlstates (
|
||||
bind_name TEXT PRIMARY KEY,
|
||||
last_page INTEGER NOT NULL DEFAULT 1,
|
||||
rnd TEXT NOT NULL DEFAULT 'a',
|
||||
error INTEGER NOT NULL DEFAULT 0,
|
||||
)
|
||||
""");
|
||||
}
|
||||
}
|
||||
|
||||
public void AddEllement(String bind, int lastpage) throws SQLException
|
||||
{
|
||||
try (PreparedStatement ps = connection.prepareStatement(insertSql))
|
||||
{
|
||||
ps.setString(1, bind);
|
||||
ps.setInt(2, lastpage);
|
||||
ps.executeUpdate();
|
||||
System.out.printf("%s is now page %d \n", bind, lastpage);
|
||||
}
|
||||
}
|
||||
|
||||
public int getPage(String bindName) throws SQLException {
|
||||
String sql = "SELECT last_page FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("last_page");
|
||||
} else {
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
public String getRND(String bindName) throws SQLException {
|
||||
String sql = "SELECT rnd FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getString("rnd");
|
||||
} else {
|
||||
return "a";
|
||||
}
|
||||
}
|
||||
}
|
||||
public int getErrors(String bindName) throws SQLException {
|
||||
String sql = "SELECT error FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("error");
|
||||
} else {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
public void updatePage(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET last_page = last_page + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void newError(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET error = error + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void resetError(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET error = 0
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void resetPage(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET last_page = 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public static String nextletter(String actuelle) {
|
||||
char[] lettres = actuelle.toCharArray();
|
||||
int i = lettres.length - 1;
|
||||
|
||||
while (i >= 0) {
|
||||
if (lettres[i] < 'z') {
|
||||
lettres[i]++;
|
||||
return new String(lettres);
|
||||
}
|
||||
lettres[i] = 'a';
|
||||
i--;
|
||||
}
|
||||
return "a" + new String(lettres);
|
||||
}
|
||||
public void updateletter(String bindName) throws SQLException
|
||||
{
|
||||
if (getErrors(bindName) >= 10)
|
||||
{
|
||||
resetError(bindName);
|
||||
resetPage(bindName);
|
||||
}
|
||||
}
|
||||
|
||||
public void Close() throws SQLException
|
||||
{
|
||||
if (connection != null)
|
||||
{
|
||||
connection.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
+150
@@ -0,0 +1,150 @@
|
||||
package org.sillyjirafe.db;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.DriverManager;
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
|
||||
public class Crawl_StateDB
|
||||
{
|
||||
String URL = new String();
|
||||
String insertSql = "INSERT OR IGNORE INTO crawlstates (bind_name, last_page, rnd, error) VALUES (?, ?, ?, ?)";
|
||||
Connection connection;
|
||||
|
||||
public Crawl_StateDB(String BDstr) throws SQLException
|
||||
{
|
||||
URL = BDstr;
|
||||
connection = DriverManager.getConnection(URL);
|
||||
}
|
||||
|
||||
public void LinkToDB() throws SQLException
|
||||
{
|
||||
try (Statement stmt = connection.createStatement())
|
||||
{
|
||||
stmt.execute("""
|
||||
CREATE TABLE IF NOT EXISTS crawlstates (
|
||||
bind_name TEXT PRIMARY KEY,
|
||||
last_page INTEGER NOT NULL DEFAULT 1,
|
||||
rnd TEXT NOT NULL DEFAULT 'a',
|
||||
error INTEGER NOT NULL DEFAULT 0,
|
||||
)
|
||||
""");
|
||||
}
|
||||
}
|
||||
|
||||
public void AddEllement(String bind, int lastpage) throws SQLException
|
||||
{
|
||||
try (PreparedStatement ps = connection.prepareStatement(insertSql))
|
||||
{
|
||||
ps.setString(1, bind);
|
||||
ps.setInt(2, lastpage);
|
||||
ps.executeUpdate();
|
||||
System.out.printf("%s is now page %d \n", bind, lastpage);
|
||||
}
|
||||
}
|
||||
|
||||
public int getPage(String bindName) throws SQLException {
|
||||
String sql = "SELECT last_page FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("last_page");
|
||||
} else {
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
public String getRND(String bindName) throws SQLException {
|
||||
String sql = "SELECT rnd FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getString("rnd");
|
||||
} else {
|
||||
return "a";
|
||||
}
|
||||
}
|
||||
}
|
||||
public int getErrors(String bindName) throws SQLException {
|
||||
String sql = "SELECT error FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("error");
|
||||
} else {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
public void updatePage(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET last_page = last_page + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void newError(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET error = error + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void resetError(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET error = 0
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void resetPage(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET last_page = 0
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void updateletter(String bindName) throws SQLException
|
||||
{
|
||||
if (getErrors(bindName) >= 10)
|
||||
{
|
||||
resetError(bindName);
|
||||
resetPage(bindName);
|
||||
}
|
||||
}
|
||||
|
||||
public void Close() throws SQLException
|
||||
{
|
||||
if (connection != null)
|
||||
{
|
||||
connection.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
+43
@@ -0,0 +1,43 @@
|
||||
package org.sillyjirafe.uoapi.binds;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.json.JSONArray;
|
||||
import org.json.JSONObject;
|
||||
import org.sillyjirafe.uoapi.Api;
|
||||
import org.sillyjirafe.uoapi.element;
|
||||
|
||||
public class OpenVerseImagesApi implements Api
|
||||
{
|
||||
public String getName()
|
||||
{
|
||||
return "OpenVerseImagesApi";
|
||||
}
|
||||
|
||||
public String getURL(int page, String apikey) {
|
||||
return "https://api.openverse.org/v1/images/?format=json&page=" + page;
|
||||
}
|
||||
|
||||
public List<element> Parse(String jsonRaw) {
|
||||
List<element> resultats = new ArrayList<>();
|
||||
|
||||
try {
|
||||
JSONObject json = new JSONObject(jsonRaw);
|
||||
JSONArray items = json.getJSONArray("results");
|
||||
|
||||
for (int i = 0; i < items.length(); i++) {
|
||||
JSONObject image = items.getJSONObject(i);
|
||||
|
||||
String titre = image.getString("title");
|
||||
String url = image.getString("url");
|
||||
String licence = image.getString("license");
|
||||
|
||||
resultats.add(new element(titre, url, licence, "openverse", "image"));
|
||||
}
|
||||
} catch (org.json.JSONException e) {
|
||||
System.out.println("error but continue : " + e.getMessage());
|
||||
}
|
||||
return resultats;
|
||||
}
|
||||
}
|
||||
+138
@@ -0,0 +1,138 @@
|
||||
package org.sillyjirafe.db;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.DriverManager;
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
|
||||
public class Crawl_StateDB
|
||||
{
|
||||
String URL = new String();
|
||||
String insertSql = "INSERT OR IGNORE INTO crawlstates (bind_name, last_page, rnd, error) VALUES (?, ?, ?, ?)";
|
||||
Connection connection;
|
||||
|
||||
public Crawl_StateDB(String BDstr) throws SQLException
|
||||
{
|
||||
URL = BDstr;
|
||||
connection = DriverManager.getConnection(URL);
|
||||
}
|
||||
|
||||
public void LinkToDB() throws SQLException
|
||||
{
|
||||
try (Statement stmt = connection.createStatement())
|
||||
{
|
||||
stmt.execute("""
|
||||
CREATE TABLE IF NOT EXISTS crawlstates (
|
||||
bind_name TEXT PRIMARY KEY,
|
||||
last_page INTEGER NOT NULL DEFAULT 1,
|
||||
rnd TEXT NOT NULL DEFAULT 'a'
|
||||
error INTEGER NOT NULL DEFAULT 0,
|
||||
)
|
||||
""");
|
||||
}
|
||||
}
|
||||
|
||||
public void AddEllement(String bind, int lastpage) throws SQLException
|
||||
{
|
||||
try (PreparedStatement ps = connection.prepareStatement(insertSql))
|
||||
{
|
||||
ps.setString(1, bind);
|
||||
ps.setInt(2, lastpage);
|
||||
ps.executeUpdate();
|
||||
System.out.printf("%s is now page %d \n", bind, lastpage);
|
||||
}
|
||||
}
|
||||
|
||||
public int getPage(String bindName) throws SQLException {
|
||||
String sql = "SELECT last_page FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("last_page");
|
||||
} else {
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
public String getRND(String bindName) throws SQLException {
|
||||
String sql = "SELECT rnd FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getString("rnd");
|
||||
} else {
|
||||
return "a";
|
||||
}
|
||||
}
|
||||
}
|
||||
public int getErrors(String bindName) throws SQLException {
|
||||
String sql = "SELECT error FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("error");
|
||||
} else {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
public void updatePage(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, rnd) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET rnd = rnd + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void newError(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET error = error + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void resetError(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET error = 0
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void updateletter(String bindName) throws SQLException
|
||||
{
|
||||
if (getErrors(bindName) >= 10)
|
||||
{
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
public void Close() throws SQLException
|
||||
{
|
||||
if (connection != null)
|
||||
{
|
||||
connection.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
+150
@@ -0,0 +1,150 @@
|
||||
package org.sillyjirafe.db;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.DriverManager;
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
|
||||
public class Crawl_StateDB
|
||||
{
|
||||
String URL = new String();
|
||||
String insertSql = "INSERT OR IGNORE INTO crawlstates (bind_name, last_page, rnd, error) VALUES (?, ?, ?, ?)";
|
||||
Connection connection;
|
||||
|
||||
public Crawl_StateDB(String BDstr) throws SQLException
|
||||
{
|
||||
URL = BDstr;
|
||||
connection = DriverManager.getConnection(URL);
|
||||
}
|
||||
|
||||
public void LinkToDB() throws SQLException
|
||||
{
|
||||
try (Statement stmt = connection.createStatement())
|
||||
{
|
||||
stmt.execute("""
|
||||
CREATE TABLE IF NOT EXISTS crawlstates (
|
||||
bind_name TEXT PRIMARY KEY,
|
||||
last_page INTEGER NOT NULL DEFAULT 1,
|
||||
rnd TEXT NOT NULL DEFAULT 'a',
|
||||
error INTEGER NOT NULL DEFAULT 0,
|
||||
)
|
||||
""");
|
||||
}
|
||||
}
|
||||
|
||||
public void AddEllement(String bind, int lastpage) throws SQLException
|
||||
{
|
||||
try (PreparedStatement ps = connection.prepareStatement(insertSql))
|
||||
{
|
||||
ps.setString(1, bind);
|
||||
ps.setInt(2, lastpage);
|
||||
ps.executeUpdate();
|
||||
System.out.printf("%s is now page %d \n", bind, lastpage);
|
||||
}
|
||||
}
|
||||
|
||||
public int getPage(String bindName) throws SQLException {
|
||||
String sql = "SELECT last_page FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("last_page");
|
||||
} else {
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
public String getRND(String bindName) throws SQLException {
|
||||
String sql = "SELECT rnd FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getString("rnd");
|
||||
} else {
|
||||
return "a";
|
||||
}
|
||||
}
|
||||
}
|
||||
public int getErrors(String bindName) throws SQLException {
|
||||
String sql = "SELECT error FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("error");
|
||||
} else {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
public void updatePage(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET last_page = last_page + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void newError(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET error = error + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void resetError(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET error = 0
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void resetPage(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET last_page = 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void updateletter(String bindName) throws SQLException
|
||||
{
|
||||
if (getErrors(bindName) >= 10)
|
||||
{
|
||||
resetError(bindName);
|
||||
resetPage(bindName);
|
||||
}
|
||||
}
|
||||
|
||||
public void Close() throws SQLException
|
||||
{
|
||||
if (connection != null)
|
||||
{
|
||||
connection.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
+43
@@ -0,0 +1,43 @@
|
||||
package org.sillyjirafe.uoapi.binds;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.json.JSONArray;
|
||||
import org.json.JSONObject;
|
||||
import org.sillyjirafe.uoapi.Api;
|
||||
import org.sillyjirafe.uoapi.element;
|
||||
|
||||
public class OpenVerseImagesApi implements Api
|
||||
{
|
||||
public String getName()
|
||||
{
|
||||
return "OpenVerseImagesApi";
|
||||
}
|
||||
|
||||
public String getURL(int page, String apikey, String rnd) {
|
||||
return "https://api.openverse.org/v1/images/?format=json&page=" + page;
|
||||
}
|
||||
|
||||
public List<element> Parse(String jsonRaw) {
|
||||
List<element> resultats = new ArrayList<>();
|
||||
|
||||
try {
|
||||
JSONObject json = new JSONObject(jsonRaw);
|
||||
JSONArray items = json.getJSONArray("results");
|
||||
|
||||
for (int i = 0; i < items.length(); i++) {
|
||||
JSONObject image = items.getJSONObject(i);
|
||||
|
||||
String titre = image.getString("title");
|
||||
String url = image.getString("url");
|
||||
String licence = image.getString("license");
|
||||
|
||||
resultats.add(new element(titre, url, licence, "openverse", "image"));
|
||||
}
|
||||
} catch (org.json.JSONException e) {
|
||||
System.out.println("error but continue : " + e.getMessage());
|
||||
}
|
||||
return resultats;
|
||||
}
|
||||
}
|
||||
+55
@@ -0,0 +1,55 @@
|
||||
package org.sillyjirafe.linker;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import org.sillyjirafe.db.Crawl_StateDB;
|
||||
import org.sillyjirafe.db.ElementsDB;
|
||||
import org.sillyjirafe.json.ApiKeys;
|
||||
import org.sillyjirafe.uoapi.*;
|
||||
|
||||
public class Main {
|
||||
public static void main(String[] args) throws Exception
|
||||
{
|
||||
Out Out = new Out();
|
||||
ElementsDB elementDB = new ElementsDB("jdbc:sqlite:elements.db");
|
||||
Crawl_StateDB crawlstateDB = new Crawl_StateDB("jdbc:sqlite:crawlstates.db");
|
||||
elementDB.LinkToDB();
|
||||
crawlstateDB.LinkToDB();
|
||||
|
||||
Map<String, Boolean> succes = Out.getSucces();
|
||||
|
||||
List<String> bindNames = Out.PreOut();
|
||||
|
||||
Map<String, Integer> pages = new HashMap<>();
|
||||
for (String name : bindNames) {
|
||||
pages.put(name, crawlstateDB.getPage(name));
|
||||
}
|
||||
|
||||
Map<String, String> rnds = new HashMap<>();
|
||||
for (String nom : bindNames) {
|
||||
rnds.put(nom, crawlstateDB.getRND(nom));
|
||||
}
|
||||
|
||||
|
||||
List<element> elements = Out.out(pages, ApiKeys.read(), rnds);
|
||||
|
||||
for (int i = 0; i < elements.size(); i++)
|
||||
{
|
||||
element el = elements.get(i);
|
||||
elementDB.AddEllement(el.title(), el.url(), el.licence(), el.source(), el.type());
|
||||
}
|
||||
|
||||
for (String name : bindNames) {
|
||||
if (succes.getOrDefault(name, false)) {
|
||||
crawlstateDB.updatePage(name);
|
||||
} else {
|
||||
crawlstateDB.addError(name);
|
||||
}
|
||||
}
|
||||
|
||||
elementDB.Close();
|
||||
crawlstateDB.Close();
|
||||
}
|
||||
}
|
||||
+49
@@ -0,0 +1,49 @@
|
||||
package org.sillyjirafe.linker;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import org.sillyjirafe.db.Crawl_StateDB;
|
||||
import org.sillyjirafe.db.ElementsDB;
|
||||
import org.sillyjirafe.json.ApiKeys;
|
||||
import org.sillyjirafe.uoapi.*;
|
||||
|
||||
public class Main {
|
||||
public static void main(String[] args) throws Exception
|
||||
{
|
||||
Out Out = new Out();
|
||||
ElementsDB elementDB = new ElementsDB("jdbc:sqlite:elements.db");
|
||||
Crawl_StateDB crawlstateDB = new Crawl_StateDB("jdbc:sqlite:crawlstates.db");
|
||||
elementDB.LinkToDB();
|
||||
crawlstateDB.LinkToDB();
|
||||
|
||||
List<String> bindNames = Out.PreOut();
|
||||
|
||||
Map<String, Integer> pages = new HashMap<>();
|
||||
for (String nom : bindNames) {
|
||||
pages.put(nom, crawlstateDB.getPage(nom));
|
||||
}
|
||||
|
||||
Map<String, String> rnds = new HashMap<>();
|
||||
for (String nom : bindNames) {
|
||||
pages.put(nom, crawlstateDB.getPage(nom));
|
||||
}
|
||||
|
||||
|
||||
List<element> elements = Out.out(pages, ApiKeys.read());
|
||||
|
||||
for (int i = 0; i < elements.size(); i++)
|
||||
{
|
||||
element el = elements.get(i);
|
||||
elementDB.AddEllement(el.title(), el.url(), el.licence(), el.source(), el.type());
|
||||
}
|
||||
|
||||
for (String name : bindNames) {
|
||||
crawlstateDB.updatePage(name);
|
||||
}
|
||||
|
||||
elementDB.Close();
|
||||
crawlstateDB.Close();
|
||||
}
|
||||
}
|
||||
+98
@@ -0,0 +1,98 @@
|
||||
package org.sillyjirafe.db;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.DriverManager;
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
|
||||
public class Crawl_StateDB
|
||||
{
|
||||
String URL = new String();
|
||||
String insertSql = "INSERT OR IGNORE INTO crawlstates (bind_name, last_page, rnd, error) VALUES (?, ?, ?, ?)";
|
||||
Connection connection;
|
||||
|
||||
public Crawl_StateDB(String BDstr) throws SQLException
|
||||
{
|
||||
URL = BDstr;
|
||||
connection = DriverManager.getConnection(URL);
|
||||
}
|
||||
|
||||
public void LinkToDB() throws SQLException
|
||||
{
|
||||
try (Statement stmt = connection.createStatement())
|
||||
{
|
||||
stmt.execute("""
|
||||
CREATE TABLE IF NOT EXISTS crawlstates (
|
||||
bind_name TEXT PRIMARY KEY,
|
||||
last_page INTEGER NOT NULL DEFAULT 1,
|
||||
rnd TEXT NOT NULL DEFAULT 'a'
|
||||
)
|
||||
""");
|
||||
}
|
||||
}
|
||||
|
||||
public void AddEllement(String bind, int lastpage) throws SQLException
|
||||
{
|
||||
try (PreparedStatement ps = connection.prepareStatement(insertSql))
|
||||
{
|
||||
ps.setString(1, bind);
|
||||
ps.setInt(2, lastpage);
|
||||
ps.executeUpdate();
|
||||
System.out.printf("%s is now page %d \n", bind, lastpage);
|
||||
}
|
||||
}
|
||||
|
||||
public int getPage(String bindName) throws SQLException {
|
||||
String sql = "SELECT last_page FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("last_page");
|
||||
} else {
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
public String getRND(String bindName) throws SQLException {
|
||||
String sql = "SELECT rnd FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getString("rnd");
|
||||
} else {
|
||||
return "a";
|
||||
}
|
||||
}
|
||||
}
|
||||
public void updatePage(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET last_page = last_page + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void updateletter(String bindName)
|
||||
{
|
||||
|
||||
}
|
||||
|
||||
public void Close() throws SQLException
|
||||
{
|
||||
if (connection != null)
|
||||
{
|
||||
connection.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
+175
@@ -0,0 +1,175 @@
|
||||
package org.sillyjirafe.db;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.DriverManager;
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
|
||||
public class Crawl_StateDB
|
||||
{
|
||||
String URL = new String();
|
||||
String insertSql = "INSERT OR IGNORE INTO crawlstates (bind_name, last_page, rnd, error) VALUES (?, ?, ?, ?)";
|
||||
Connection connection;
|
||||
|
||||
public Crawl_StateDB(String BDstr) throws SQLException
|
||||
{
|
||||
URL = BDstr;
|
||||
connection = DriverManager.getConnection(URL);
|
||||
}
|
||||
|
||||
public void LinkToDB() throws SQLException
|
||||
{
|
||||
try (Statement stmt = connection.createStatement())
|
||||
{
|
||||
stmt.execute("""
|
||||
CREATE TABLE IF NOT EXISTS crawlstates (
|
||||
bind_name TEXT PRIMARY KEY,
|
||||
last_page INTEGER NOT NULL DEFAULT 1,
|
||||
rnd TEXT NOT NULL DEFAULT 'a',
|
||||
error INTEGER NOT NULL DEFAULT 0,
|
||||
)
|
||||
""");
|
||||
}
|
||||
}
|
||||
|
||||
public void AddEllement(String bind, int lastpage) throws SQLException
|
||||
{
|
||||
try (PreparedStatement ps = connection.prepareStatement(insertSql))
|
||||
{
|
||||
ps.setString(1, bind);
|
||||
ps.setInt(2, lastpage);
|
||||
ps.executeUpdate();
|
||||
System.out.printf("%s is now page %d \n", bind, lastpage);
|
||||
}
|
||||
}
|
||||
|
||||
public int getPage(String bindName) throws SQLException {
|
||||
String sql = "SELECT last_page FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("last_page");
|
||||
} else {
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
public String getRND(String bindName) throws SQLException {
|
||||
String sql = "SELECT rnd FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getString("rnd");
|
||||
} else {
|
||||
return "a";
|
||||
}
|
||||
}
|
||||
}
|
||||
public int getErrors(String bindName) throws SQLException {
|
||||
String sql = "SELECT error FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("error");
|
||||
} else {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
public void updatePage(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET last_page = last_page + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void setRnd(String bindName, String c) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET last_page = last_page + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void newError(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET error = error + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void resetError(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET error = 0
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void resetPage(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET last_page = 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public static String nextletter(String current) {
|
||||
char[] lettres = current.toCharArray();
|
||||
int i = lettres.length - 1;
|
||||
|
||||
while (i >= 0) {
|
||||
if (lettres[i] < 'z') {
|
||||
lettres[i]++;
|
||||
return new String(lettres);
|
||||
}
|
||||
lettres[i] = 'a';
|
||||
i--;
|
||||
}
|
||||
return "a" + new String(lettres);
|
||||
}
|
||||
public void updateletter(String bindName) throws SQLException
|
||||
{
|
||||
if (getErrors(bindName) >= 10)
|
||||
{
|
||||
resetError(bindName);
|
||||
resetPage(bindName);
|
||||
}
|
||||
}
|
||||
|
||||
public void Close() throws SQLException
|
||||
{
|
||||
if (connection != null)
|
||||
{
|
||||
connection.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
+175
@@ -0,0 +1,175 @@
|
||||
package org.sillyjirafe.db;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.DriverManager;
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
|
||||
public class Crawl_StateDB
|
||||
{
|
||||
String URL = new String();
|
||||
String insertSql = "INSERT OR IGNORE INTO crawlstates (bind_name, last_page, rnd, error) VALUES (?, ?, ?, ?)";
|
||||
Connection connection;
|
||||
|
||||
public Crawl_StateDB(String BDstr) throws SQLException
|
||||
{
|
||||
URL = BDstr;
|
||||
connection = DriverManager.getConnection(URL);
|
||||
}
|
||||
|
||||
public void LinkToDB() throws SQLException
|
||||
{
|
||||
try (Statement stmt = connection.createStatement())
|
||||
{
|
||||
stmt.execute("""
|
||||
CREATE TABLE IF NOT EXISTS crawlstates (
|
||||
bind_name TEXT PRIMARY KEY,
|
||||
last_page INTEGER NOT NULL DEFAULT 1,
|
||||
rnd TEXT NOT NULL DEFAULT 'a',
|
||||
error INTEGER NOT NULL DEFAULT 0,
|
||||
)
|
||||
""");
|
||||
}
|
||||
}
|
||||
|
||||
public void AddEllement(String bind, int lastpage) throws SQLException
|
||||
{
|
||||
try (PreparedStatement ps = connection.prepareStatement(insertSql))
|
||||
{
|
||||
ps.setString(1, bind);
|
||||
ps.setInt(2, lastpage);
|
||||
ps.executeUpdate();
|
||||
System.out.printf("%s is now page %d \n", bind, lastpage);
|
||||
}
|
||||
}
|
||||
|
||||
public int getPage(String bindName) throws SQLException {
|
||||
String sql = "SELECT last_page FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("last_page");
|
||||
} else {
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
public String getRND(String bindName) throws SQLException {
|
||||
String sql = "SELECT rnd FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getString("rnd");
|
||||
} else {
|
||||
return "a";
|
||||
}
|
||||
}
|
||||
}
|
||||
public int getErrors(String bindName) throws SQLException {
|
||||
String sql = "SELECT error FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("error");
|
||||
} else {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
public void updatePage(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET last_page = last_page + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void setRnd(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET last_page = last_page + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void newError(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET error = error + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void resetError(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET error = 0
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void resetPage(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET last_page = 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public static String nextletter(String current) {
|
||||
char[] lettres = current.toCharArray();
|
||||
int i = lettres.length - 1;
|
||||
|
||||
while (i >= 0) {
|
||||
if (lettres[i] < 'z') {
|
||||
lettres[i]++;
|
||||
return new String(lettres);
|
||||
}
|
||||
lettres[i] = 'a';
|
||||
i--;
|
||||
}
|
||||
return "a" + new String(lettres);
|
||||
}
|
||||
public void updateletter(String bindName) throws SQLException
|
||||
{
|
||||
if (getErrors(bindName) >= 10)
|
||||
{
|
||||
resetError(bindName);
|
||||
resetPage(bindName);
|
||||
}
|
||||
}
|
||||
|
||||
public void Close() throws SQLException
|
||||
{
|
||||
if (connection != null)
|
||||
{
|
||||
connection.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
+46
@@ -0,0 +1,46 @@
|
||||
package org.sillyjirafe.uoapi;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import org.sillyjirafe.uoapi.binds.*;
|
||||
|
||||
public class Out {
|
||||
List<Api> Apis = new ArrayList<>();
|
||||
|
||||
public List<String> PreOut()
|
||||
{
|
||||
Apis.add(new GitHubMITApi());
|
||||
Apis.add(new GitHubLGPLApi());
|
||||
Apis.add(new GitHubApacheApi());
|
||||
Apis.add(new OpenVerseImagesApi());
|
||||
Apis.add(new OpenVerseAudiosApi());
|
||||
Apis.add(new PixabayImagesApi());
|
||||
|
||||
List<String> bindnames = new ArrayList<>();
|
||||
|
||||
for (Api api : Apis)
|
||||
{
|
||||
bindnames.add(api.getName());
|
||||
}
|
||||
|
||||
return bindnames;
|
||||
}
|
||||
|
||||
public List<element> out(Map<String, Integer> bindpages, Map<String, String> ApiKeys, Map<String, String> Rnds) throws Exception
|
||||
{
|
||||
List<element> Elements = new ArrayList<>();
|
||||
|
||||
for (int i = 0; i < Apis.size(); i++)
|
||||
{
|
||||
List<element> currentapi = Caller.CallApi(Apis.get(i), bindpages.get(Apis.get(i).getName()), ApiKeys.get(Apis.get(i).getName()), );
|
||||
for (int j = 0; j < currentapi.size(); j++)
|
||||
{
|
||||
Elements.add(currentapi.get(j));
|
||||
}
|
||||
}
|
||||
return Elements;
|
||||
}
|
||||
}
|
||||
+164
@@ -0,0 +1,164 @@
|
||||
package org.sillyjirafe.db;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.DriverManager;
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
|
||||
public class Crawl_StateDB
|
||||
{
|
||||
String URL = new String();
|
||||
String insertSql = "INSERT OR IGNORE INTO crawlstates (bind_name, last_page, rnd, error) VALUES (?, ?, ?, ?)";
|
||||
Connection connection;
|
||||
|
||||
public Crawl_StateDB(String BDstr) throws SQLException
|
||||
{
|
||||
URL = BDstr;
|
||||
connection = DriverManager.getConnection(URL);
|
||||
}
|
||||
|
||||
public void LinkToDB() throws SQLException
|
||||
{
|
||||
try (Statement stmt = connection.createStatement())
|
||||
{
|
||||
stmt.execute("""
|
||||
CREATE TABLE IF NOT EXISTS crawlstates (
|
||||
bind_name TEXT PRIMARY KEY,
|
||||
last_page INTEGER NOT NULL DEFAULT 1,
|
||||
rnd TEXT NOT NULL DEFAULT 'a',
|
||||
error INTEGER NOT NULL DEFAULT 0,
|
||||
)
|
||||
""");
|
||||
}
|
||||
}
|
||||
|
||||
public void AddEllement(String bind, int lastpage) throws SQLException
|
||||
{
|
||||
try (PreparedStatement ps = connection.prepareStatement(insertSql))
|
||||
{
|
||||
ps.setString(1, bind);
|
||||
ps.setInt(2, lastpage);
|
||||
ps.executeUpdate();
|
||||
System.out.printf("%s is now page %d \n", bind, lastpage);
|
||||
}
|
||||
}
|
||||
|
||||
public int getPage(String bindName) throws SQLException {
|
||||
String sql = "SELECT last_page FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("last_page");
|
||||
} else {
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
public String getRND(String bindName) throws SQLException {
|
||||
String sql = "SELECT rnd FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getString("rnd");
|
||||
} else {
|
||||
return "a";
|
||||
}
|
||||
}
|
||||
}
|
||||
public int getErrors(String bindName) throws SQLException {
|
||||
String sql = "SELECT error FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("error");
|
||||
} else {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
public void updatePage(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET last_page = last_page + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void newError(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET error = error + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void resetError(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET error = 0
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void resetPage(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET last_page = 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public static String lettreSuivante(String actuelle) {
|
||||
char[] lettres = actuelle.toCharArray();
|
||||
int i = lettres.length - 1;
|
||||
|
||||
while (i >= 0) {
|
||||
if (lettres[i] < 'z') {
|
||||
lettres[i]++;
|
||||
return new String(lettres);
|
||||
}
|
||||
lettres[i] = 'a';
|
||||
i--;
|
||||
}
|
||||
return "a" + new String(lettres);
|
||||
}
|
||||
public void updateletter(String bindName) throws SQLException
|
||||
{
|
||||
if (getErrors(bindName) >= 10)
|
||||
{
|
||||
resetError(bindName);
|
||||
resetPage(bindName);
|
||||
}
|
||||
}
|
||||
|
||||
public void Close() throws SQLException
|
||||
{
|
||||
if (connection != null)
|
||||
{
|
||||
connection.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
+99
@@ -0,0 +1,99 @@
|
||||
package org.sillyjirafe.db;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.DriverManager;
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
|
||||
public class Crawl_StateDB
|
||||
{
|
||||
String URL = new String();
|
||||
String insertSql = "INSERT OR IGNORE INTO crawlstates (bind_name, last_page, rnd, error) VALUES (?, ?, ?, ?)";
|
||||
Connection connection;
|
||||
|
||||
public Crawl_StateDB(String BDstr) throws SQLException
|
||||
{
|
||||
URL = BDstr;
|
||||
connection = DriverManager.getConnection(URL);
|
||||
}
|
||||
|
||||
public void LinkToDB() throws SQLException
|
||||
{
|
||||
try (Statement stmt = connection.createStatement())
|
||||
{
|
||||
stmt.execute("""
|
||||
CREATE TABLE IF NOT EXISTS crawlstates (
|
||||
bind_name TEXT PRIMARY KEY,
|
||||
last_page INTEGER NOT NULL DEFAULT 1,
|
||||
rnd TEXT NOT NULL DEFAULT 'a'
|
||||
error INTEGER NOT NULL DEFAULT 0,
|
||||
)
|
||||
""");
|
||||
}
|
||||
}
|
||||
|
||||
public void AddEllement(String bind, int lastpage) throws SQLException
|
||||
{
|
||||
try (PreparedStatement ps = connection.prepareStatement(insertSql))
|
||||
{
|
||||
ps.setString(1, bind);
|
||||
ps.setInt(2, lastpage);
|
||||
ps.executeUpdate();
|
||||
System.out.printf("%s is now page %d \n", bind, lastpage);
|
||||
}
|
||||
}
|
||||
|
||||
public int getPage(String bindName) throws SQLException {
|
||||
String sql = "SELECT last_page FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("last_page");
|
||||
} else {
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
public String getRND(String bindName) throws SQLException {
|
||||
String sql = "SELECT rnd FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getString("rnd");
|
||||
} else {
|
||||
return "a";
|
||||
}
|
||||
}
|
||||
}
|
||||
public void updatePage(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET last_page = last_page + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void updateletter(String bindName)
|
||||
{
|
||||
|
||||
}
|
||||
|
||||
public void Close() throws SQLException
|
||||
{
|
||||
if (connection != null)
|
||||
{
|
||||
connection.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
+177
@@ -0,0 +1,177 @@
|
||||
package org.sillyjirafe.db;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.DriverManager;
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
|
||||
public class Crawl_StateDB
|
||||
{
|
||||
String URL = new String();
|
||||
String insertSql = "INSERT OR IGNORE INTO crawlstates (bind_name, last_page, rnd, error) VALUES (?, ?, ?, ?)";
|
||||
Connection connection;
|
||||
|
||||
public Crawl_StateDB(String BDstr) throws SQLException
|
||||
{
|
||||
URL = BDstr;
|
||||
connection = DriverManager.getConnection(URL);
|
||||
}
|
||||
|
||||
public void LinkToDB() throws SQLException
|
||||
{
|
||||
try (Statement stmt = connection.createStatement())
|
||||
{
|
||||
stmt.execute("""
|
||||
CREATE TABLE IF NOT EXISTS crawlstates (
|
||||
bind_name TEXT PRIMARY KEY,
|
||||
last_page INTEGER NOT NULL DEFAULT 1,
|
||||
rnd TEXT NOT NULL DEFAULT 'a',
|
||||
error INTEGER NOT NULL DEFAULT 0,
|
||||
)
|
||||
""");
|
||||
}
|
||||
}
|
||||
|
||||
public void AddEllement(String bind, int lastpage) throws SQLException
|
||||
{
|
||||
try (PreparedStatement ps = connection.prepareStatement(insertSql))
|
||||
{
|
||||
ps.setString(1, bind);
|
||||
ps.setInt(2, lastpage);
|
||||
ps.executeUpdate();
|
||||
System.out.printf("%s is now page %d \n", bind, lastpage);
|
||||
}
|
||||
}
|
||||
|
||||
public int getPage(String bindName) throws SQLException {
|
||||
String sql = "SELECT last_page FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("last_page");
|
||||
} else {
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
public String getRND(String bindName) throws SQLException {
|
||||
String sql = "SELECT rnd FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getString("rnd");
|
||||
} else {
|
||||
return "a";
|
||||
}
|
||||
}
|
||||
}
|
||||
public int getErrors(String bindName) throws SQLException {
|
||||
String sql = "SELECT error FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("error");
|
||||
} else {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
public void updatePage(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET last_page = last_page + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void setRnd(String bindName, String c) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, rnd) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET rnd = (rnd)
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.setString(2, c);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void newError(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET error = error + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void resetError(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET error = 0
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void resetPage(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET last_page = 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public static String nextLetter(String current) {
|
||||
char[] lettres = current.toCharArray();
|
||||
int i = lettres.length - 1;
|
||||
|
||||
while (i >= 0) {
|
||||
if (lettres[i] < 'z') {
|
||||
lettres[i]++;
|
||||
return new String(lettres);
|
||||
}
|
||||
lettres[i] = 'a';
|
||||
i--;
|
||||
}
|
||||
return "a" + new String(lettres);
|
||||
}
|
||||
public void updateletter(String bindName) throws SQLException
|
||||
{
|
||||
if (getErrors(bindName) >= 10)
|
||||
{
|
||||
setRnd(bindName, nextLetter(getRND(bindName)));
|
||||
resetError(bindName);
|
||||
resetPage(bindName);
|
||||
}
|
||||
}
|
||||
|
||||
public void Close() throws SQLException
|
||||
{
|
||||
if (connection != null)
|
||||
{
|
||||
connection.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
+50
@@ -0,0 +1,50 @@
|
||||
package org.sillyjirafe.uoapi;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import org.sillyjirafe.uoapi.binds.*;
|
||||
|
||||
public class Out {
|
||||
List<Api> Apis = new ArrayList<>();
|
||||
Map<String, Boolean> Succes = new HashMap<>();
|
||||
|
||||
public List<String> PreOut()
|
||||
{
|
||||
Apis.add(new GitHubMITApi());
|
||||
Apis.add(new GitHubLGPLApi());
|
||||
Apis.add(new GitHubApacheApi());
|
||||
Apis.add(new OpenVerseImagesApi());
|
||||
Apis.add(new OpenVerseAudiosApi());
|
||||
Apis.add(new PixabayImagesApi());
|
||||
|
||||
List<String> bindnames = new ArrayList<>();
|
||||
|
||||
for (Api api : Apis)
|
||||
{
|
||||
bindnames.add(api.getName());
|
||||
}
|
||||
|
||||
return bindnames;
|
||||
}
|
||||
|
||||
public List<element> out(Map<String, Integer> bindpages, Map<String, String> ApiKeys, Map<String, String> Rnds) throws Exception
|
||||
{
|
||||
List<element> Elements = new ArrayList<>();
|
||||
|
||||
|
||||
for (Api api : Apis) {
|
||||
try {
|
||||
List<element> currentapi = Caller.CallApi(api, bindpages.get(api.getName()));
|
||||
Elements.addAll(currentapi);
|
||||
Succes.put(api.getName(), true);
|
||||
} catch (Exception e) {
|
||||
System.out.println("Erreur pour " + api.getName() + " : " + e.getMessage());
|
||||
Succes.put(api.getName(), false);
|
||||
}
|
||||
}
|
||||
return Elements;
|
||||
}
|
||||
}
|
||||
+45
@@ -0,0 +1,45 @@
|
||||
package org.sillyjirafe.uoapi.binds;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.json.JSONArray;
|
||||
import org.json.JSONObject;
|
||||
import org.sillyjirafe.uoapi.Api;
|
||||
import org.sillyjirafe.uoapi.element;
|
||||
|
||||
public class GitHubLGPLApi implements Api
|
||||
{
|
||||
|
||||
public String getName()
|
||||
{
|
||||
return "GitHubLGPLApi";
|
||||
}
|
||||
|
||||
public String getURL(int page, String apikey) {
|
||||
return "https://api.github.com/search/repositories?q=license:lgpl-3.0&page=" + page;
|
||||
}
|
||||
|
||||
public List<element> Parse(String jsonRaw) {
|
||||
List<element> resultats = new ArrayList<>();
|
||||
|
||||
try {
|
||||
JSONObject json = new JSONObject(jsonRaw);
|
||||
JSONArray items = json.getJSONArray("items");
|
||||
|
||||
for (int i = 0; i < items.length(); i++) {
|
||||
JSONObject repo = items.getJSONObject(i);
|
||||
|
||||
String titre = repo.getString("full_name");
|
||||
String url = repo.getString("html_url");
|
||||
String licence = repo.getJSONObject("license").getString("spdx_id");
|
||||
|
||||
resultats.add(new element(titre, url, licence, "github", "code"));
|
||||
}
|
||||
} catch (org.json.JSONException e) {
|
||||
System.out.println("error but continue : " + e.getMessage());
|
||||
}
|
||||
|
||||
return resultats;
|
||||
}
|
||||
}
|
||||
+50
@@ -0,0 +1,50 @@
|
||||
package org.sillyjirafe.uoapi;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import org.sillyjirafe.uoapi.binds.*;
|
||||
|
||||
public class Out {
|
||||
List<Api> Apis = new ArrayList<>();
|
||||
Map<String, Boolean> Succes = new HashMap<>();
|
||||
|
||||
public List<String> PreOut()
|
||||
{
|
||||
Apis.add(new GitHubMITApi());
|
||||
Apis.add(new GitHubLGPLApi());
|
||||
Apis.add(new GitHubApacheApi());
|
||||
Apis.add(new OpenVerseImagesApi());
|
||||
Apis.add(new OpenVerseAudiosApi());
|
||||
Apis.add(new PixabayImagesApi());
|
||||
|
||||
List<String> bindnames = new ArrayList<>();
|
||||
|
||||
for (Api api : Apis)
|
||||
{
|
||||
bindnames.add(api.getName());
|
||||
}
|
||||
|
||||
return bindnames;
|
||||
}
|
||||
|
||||
public List<element> out(Map<String, Integer> bindpages, Map<String, String> ApiKeys, Map<String, String> Rnds) throws Exception
|
||||
{
|
||||
List<element> Elements = new ArrayList<>();
|
||||
|
||||
|
||||
for (Api api : Apis) {
|
||||
try {
|
||||
List<element> currentapi = Caller.CallApi(api, bindpages.get(api.getName()), ApiKeys.get(api.getName()), Rnds.get(api.getName()));
|
||||
Elements.addAll(currentapi);
|
||||
Succes.put(api.getName(), true);
|
||||
} catch (Exception e) {
|
||||
System.out.println("Erreur pour " + api.getName() + " : " + e.getMessage());
|
||||
Succes.put(api.getName(), false);
|
||||
}
|
||||
}
|
||||
return Elements;
|
||||
}
|
||||
}
|
||||
+44
@@ -0,0 +1,44 @@
|
||||
package org.sillyjirafe.linker;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import org.sillyjirafe.db.Crawl_StateDB;
|
||||
import org.sillyjirafe.db.ElementsDB;
|
||||
import org.sillyjirafe.json.ApiKeys;
|
||||
import org.sillyjirafe.uoapi.*;
|
||||
|
||||
public class Main {
|
||||
public static void main(String[] args) throws Exception
|
||||
{
|
||||
Out Out = new Out();
|
||||
ElementsDB elementDB = new ElementsDB("jdbc:sqlite:elements.db");
|
||||
Crawl_StateDB crawlstateDB = new Crawl_StateDB("jdbc:sqlite:crawlstates.db");
|
||||
elementDB.LinkToDB();
|
||||
crawlstateDB.LinkToDB();
|
||||
|
||||
List<String> bindNames = Out.PreOut();
|
||||
|
||||
Map<String, Integer> pages = new HashMap<>();
|
||||
for (String nom : bindNames) {
|
||||
pages.put(nom, crawlstateDB.getPage(nom));
|
||||
}
|
||||
|
||||
|
||||
List<element> elements = Out.out(pages, ApiKeys.read());
|
||||
|
||||
for (int i = 0; i < elements.size(); i++)
|
||||
{
|
||||
element el = elements.get(i);
|
||||
elementDB.AddEllement(el.title(), el.url(), el.licence(), el.source(), el.type());
|
||||
}
|
||||
|
||||
for (String name : bindNames) {
|
||||
crawlstateDB.updatePage(name);
|
||||
}
|
||||
|
||||
elementDB.Close();
|
||||
crawlstateDB.Close();
|
||||
}
|
||||
}
|
||||
+45
@@ -0,0 +1,45 @@
|
||||
package org.sillyjirafe.uoapi.binds;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.json.JSONArray;
|
||||
import org.json.JSONObject;
|
||||
import org.sillyjirafe.uoapi.Api;
|
||||
import org.sillyjirafe.uoapi.element;
|
||||
|
||||
public class GitHubLGPLApi implements Api
|
||||
{
|
||||
|
||||
public String getName()
|
||||
{
|
||||
return "GitHubLGPLApi";
|
||||
}
|
||||
|
||||
public String getURL(int page, String apikey, String rnd) {
|
||||
return "https://api.github.com/search/repositories?q=license:lgpl-3.0&page=" + page;
|
||||
}
|
||||
|
||||
public List<element> Parse(String jsonRaw) {
|
||||
List<element> resultats = new ArrayList<>();
|
||||
|
||||
try {
|
||||
JSONObject json = new JSONObject(jsonRaw);
|
||||
JSONArray items = json.getJSONArray("items");
|
||||
|
||||
for (int i = 0; i < items.length(); i++) {
|
||||
JSONObject repo = items.getJSONObject(i);
|
||||
|
||||
String titre = repo.getString("full_name");
|
||||
String url = repo.getString("html_url");
|
||||
String licence = repo.getJSONObject("license").getString("spdx_id");
|
||||
|
||||
resultats.add(new element(titre, url, licence, "github", "code"));
|
||||
}
|
||||
} catch (org.json.JSONException e) {
|
||||
System.out.println("error but continue : " + e.getMessage());
|
||||
}
|
||||
|
||||
return resultats;
|
||||
}
|
||||
}
|
||||
+177
@@ -0,0 +1,177 @@
|
||||
package org.sillyjirafe.db;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.DriverManager;
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
|
||||
public class Crawl_StateDB
|
||||
{
|
||||
String URL = new String();
|
||||
String insertSql = "INSERT OR IGNORE INTO crawlstates (bind_name, last_page, rnd, error) VALUES (?, ?, ?, ?)";
|
||||
Connection connection;
|
||||
|
||||
public Crawl_StateDB(String BDstr) throws SQLException
|
||||
{
|
||||
URL = BDstr;
|
||||
connection = DriverManager.getConnection(URL);
|
||||
}
|
||||
|
||||
public void LinkToDB() throws SQLException
|
||||
{
|
||||
try (Statement stmt = connection.createStatement())
|
||||
{
|
||||
stmt.execute("""
|
||||
CREATE TABLE IF NOT EXISTS crawlstates (
|
||||
bind_name TEXT PRIMARY KEY,
|
||||
last_page INTEGER NOT NULL DEFAULT 1,
|
||||
rnd TEXT NOT NULL DEFAULT 'a',
|
||||
error INTEGER NOT NULL DEFAULT 0,
|
||||
)
|
||||
""");
|
||||
}
|
||||
}
|
||||
|
||||
public void AddEllement(String bind, int lastpage) throws SQLException
|
||||
{
|
||||
try (PreparedStatement ps = connection.prepareStatement(insertSql))
|
||||
{
|
||||
ps.setString(1, bind);
|
||||
ps.setInt(2, lastpage);
|
||||
ps.executeUpdate();
|
||||
System.out.printf("%s is now page %d \n", bind, lastpage);
|
||||
}
|
||||
}
|
||||
|
||||
public int getPage(String bindName) throws SQLException {
|
||||
String sql = "SELECT last_page FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("last_page");
|
||||
} else {
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
public String getRND(String bindName) throws SQLException {
|
||||
String sql = "SELECT rnd FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getString("rnd");
|
||||
} else {
|
||||
return "a";
|
||||
}
|
||||
}
|
||||
}
|
||||
public int getErrors(String bindName) throws SQLException {
|
||||
String sql = "SELECT error FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("error");
|
||||
} else {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
public void updatePage(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET last_page = last_page + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void setRnd(String bindName, String c) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, rnd) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET rnd = (rnd)
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void newError(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET error = error + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void resetError(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET error = 0
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void resetPage(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET last_page = 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public static String nextLetter(String current) {
|
||||
char[] lettres = current.toCharArray();
|
||||
int i = lettres.length - 1;
|
||||
|
||||
while (i >= 0) {
|
||||
if (lettres[i] < 'z') {
|
||||
lettres[i]++;
|
||||
return new String(lettres);
|
||||
}
|
||||
lettres[i] = 'a';
|
||||
i--;
|
||||
}
|
||||
return "a" + new String(lettres);
|
||||
}
|
||||
public void updateletter(String bindName) throws SQLException
|
||||
{
|
||||
if (getErrors(bindName) >= 10)
|
||||
{
|
||||
setRnd(bindName, nextLetter(getRND(bindName)));
|
||||
resetError(bindName);
|
||||
resetPage(bindName);
|
||||
}
|
||||
}
|
||||
|
||||
public void Close() throws SQLException
|
||||
{
|
||||
if (connection != null)
|
||||
{
|
||||
connection.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
+45
@@ -0,0 +1,45 @@
|
||||
package org.sillyjirafe.uoapi.binds;
|
||||
|
||||
import org.json.JSONArray;
|
||||
import org.json.JSONObject;
|
||||
import org.sillyjirafe.uoapi.Api;
|
||||
import org.sillyjirafe.uoapi.element;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
public class GitHubMITApi implements Api
|
||||
{
|
||||
|
||||
public String getName()
|
||||
{
|
||||
return "GitHubMITApi";
|
||||
}
|
||||
|
||||
public String getURL(int page, String apikey) {
|
||||
return "https://api.github.com/search/repositories?q=license:mit&page=" + page;
|
||||
}
|
||||
|
||||
public List<element> Parse(String jsonRaw) {
|
||||
List<element> resultats = new ArrayList<>();
|
||||
|
||||
|
||||
try {
|
||||
JSONObject json = new JSONObject(jsonRaw);
|
||||
JSONArray items = json.getJSONArray("items");
|
||||
|
||||
for (int i = 0; i < items.length(); i++) {
|
||||
JSONObject repo = items.getJSONObject(i);
|
||||
|
||||
String titre = repo.getString("full_name");
|
||||
String url = repo.getString("html_url");
|
||||
String licence = repo.getJSONObject("license").getString("spdx_id");
|
||||
|
||||
resultats.add(new element(titre, url, licence, "github", "code"));
|
||||
}
|
||||
} catch (org.json.JSONException e) {
|
||||
System.out.println("error but continue : " + e.getMessage());
|
||||
}
|
||||
|
||||
return resultats;
|
||||
}
|
||||
}
|
||||
+139
@@ -0,0 +1,139 @@
|
||||
package org.sillyjirafe.db;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.DriverManager;
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
|
||||
public class Crawl_StateDB
|
||||
{
|
||||
String URL = new String();
|
||||
String insertSql = "INSERT OR IGNORE INTO crawlstates (bind_name, last_page, rnd, error) VALUES (?, ?, ?, ?)";
|
||||
Connection connection;
|
||||
|
||||
public Crawl_StateDB(String BDstr) throws SQLException
|
||||
{
|
||||
URL = BDstr;
|
||||
connection = DriverManager.getConnection(URL);
|
||||
}
|
||||
|
||||
public void LinkToDB() throws SQLException
|
||||
{
|
||||
try (Statement stmt = connection.createStatement())
|
||||
{
|
||||
stmt.execute("""
|
||||
CREATE TABLE IF NOT EXISTS crawlstates (
|
||||
bind_name TEXT PRIMARY KEY,
|
||||
last_page INTEGER NOT NULL DEFAULT 1,
|
||||
rnd TEXT NOT NULL DEFAULT 'a'
|
||||
error INTEGER NOT NULL DEFAULT 0,
|
||||
)
|
||||
""");
|
||||
}
|
||||
}
|
||||
|
||||
public void AddEllement(String bind, int lastpage) throws SQLException
|
||||
{
|
||||
try (PreparedStatement ps = connection.prepareStatement(insertSql))
|
||||
{
|
||||
ps.setString(1, bind);
|
||||
ps.setInt(2, lastpage);
|
||||
ps.executeUpdate();
|
||||
System.out.printf("%s is now page %d \n", bind, lastpage);
|
||||
}
|
||||
}
|
||||
|
||||
public int getPage(String bindName) throws SQLException {
|
||||
String sql = "SELECT last_page FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("last_page");
|
||||
} else {
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
public String getRND(String bindName) throws SQLException {
|
||||
String sql = "SELECT rnd FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getString("rnd");
|
||||
} else {
|
||||
return "a";
|
||||
}
|
||||
}
|
||||
}
|
||||
public int getErrors(String bindName) throws SQLException {
|
||||
String sql = "SELECT error FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("error");
|
||||
} else {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
public void updatePage(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, rnd) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET rnd = rnd + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void newError(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET error = error + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void resetError(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET error = 0
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void updateletter(String bindName) throws SQLException
|
||||
{
|
||||
if (getErrors(bindName) >= 10)
|
||||
{
|
||||
resetError(bindName);
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
public void Close() throws SQLException
|
||||
{
|
||||
if (connection != null)
|
||||
{
|
||||
connection.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
+49
@@ -0,0 +1,49 @@
|
||||
package org.sillyjirafe.linker;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import org.sillyjirafe.db.Crawl_StateDB;
|
||||
import org.sillyjirafe.db.ElementsDB;
|
||||
import org.sillyjirafe.json.ApiKeys;
|
||||
import org.sillyjirafe.uoapi.*;
|
||||
|
||||
public class Main {
|
||||
public static void main(String[] args) throws Exception
|
||||
{
|
||||
Out Out = new Out();
|
||||
ElementsDB elementDB = new ElementsDB("jdbc:sqlite:elements.db");
|
||||
Crawl_StateDB crawlstateDB = new Crawl_StateDB("jdbc:sqlite:crawlstates.db");
|
||||
elementDB.LinkToDB();
|
||||
crawlstateDB.LinkToDB();
|
||||
|
||||
List<String> bindNames = Out.PreOut();
|
||||
|
||||
Map<String, Integer> pages = new HashMap<>();
|
||||
for (String nom : bindNames) {
|
||||
pages.put(nom, crawlstateDB.getPage(nom));
|
||||
}
|
||||
|
||||
Map<String, String> rnds = new HashMap<>();
|
||||
for (String nom : bindNames) {
|
||||
rnds.put(nom, crawlstateDB.getRND(nom));
|
||||
}
|
||||
|
||||
|
||||
List<element> elements = Out.out(pages, ApiKeys.read(), rnds);
|
||||
|
||||
for (int i = 0; i < elements.size(); i++)
|
||||
{
|
||||
element el = elements.get(i);
|
||||
elementDB.AddEllement(el.title(), el.url(), el.licence(), el.source(), el.type());
|
||||
}
|
||||
|
||||
for (String name : bindNames) {
|
||||
crawlstateDB.updatePage(name);
|
||||
}
|
||||
|
||||
elementDB.Close();
|
||||
crawlstateDB.Close();
|
||||
}
|
||||
}
|
||||
+55
@@ -0,0 +1,55 @@
|
||||
package org.sillyjirafe.linker;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import org.sillyjirafe.db.Crawl_StateDB;
|
||||
import org.sillyjirafe.db.ElementsDB;
|
||||
import org.sillyjirafe.json.ApiKeys;
|
||||
import org.sillyjirafe.uoapi.*;
|
||||
|
||||
public class Main {
|
||||
public static void main(String[] args) throws Exception
|
||||
{
|
||||
Out Out = new Out();
|
||||
ElementsDB elementDB = new ElementsDB("jdbc:sqlite:elements.db");
|
||||
Crawl_StateDB crawlstateDB = new Crawl_StateDB("jdbc:sqlite:crawlstates.db");
|
||||
elementDB.LinkToDB();
|
||||
crawlstateDB.LinkToDB();
|
||||
|
||||
Map<String, Boolean> succes = Out.getSucces();
|
||||
|
||||
List<String> bindNames = Out.PreOut();
|
||||
|
||||
Map<String, Integer> pages = new HashMap<>();
|
||||
for (String nom : bindNames) {
|
||||
pages.put(nom, crawlstateDB.getPage(nom));
|
||||
}
|
||||
|
||||
Map<String, String> rnds = new HashMap<>();
|
||||
for (String nom : bindNames) {
|
||||
rnds.put(nom, crawlstateDB.getRND(nom));
|
||||
}
|
||||
|
||||
|
||||
List<element> elements = Out.out(pages, ApiKeys.read(), rnds);
|
||||
|
||||
for (int i = 0; i < elements.size(); i++)
|
||||
{
|
||||
element el = elements.get(i);
|
||||
elementDB.AddEllement(el.title(), el.url(), el.licence(), el.source(), el.type());
|
||||
}
|
||||
|
||||
for (String name : bindNames) {
|
||||
if (Succes.getOrDefault(nom, false)) {
|
||||
crawlstateDB.updatePage(nom, ...); // succès normal, avance
|
||||
} else {
|
||||
crawlstateDB.incrementError(nom); // échec, compte une erreur de plus
|
||||
}
|
||||
}
|
||||
|
||||
elementDB.Close();
|
||||
crawlstateDB.Close();
|
||||
}
|
||||
}
|
||||
+45
@@ -0,0 +1,45 @@
|
||||
package org.sillyjirafe.uoapi.binds;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.json.JSONArray;
|
||||
import org.json.JSONObject;
|
||||
import org.sillyjirafe.uoapi.Api;
|
||||
import org.sillyjirafe.uoapi.element;
|
||||
|
||||
public class GitHubLGPLApi implements Api
|
||||
{
|
||||
|
||||
public String getName()
|
||||
{
|
||||
return "GitHubLGPLApi";
|
||||
}
|
||||
|
||||
public String getURL(int page, String apikey, String rnd) {
|
||||
return "https://api.github.com/search/repositories?q="+ rnd + "&license:lgpl-3.0&page=" + page;
|
||||
}
|
||||
|
||||
public List<element> Parse(String jsonRaw) {
|
||||
List<element> resultats = new ArrayList<>();
|
||||
|
||||
try {
|
||||
JSONObject json = new JSONObject(jsonRaw);
|
||||
JSONArray items = json.getJSONArray("items");
|
||||
|
||||
for (int i = 0; i < items.length(); i++) {
|
||||
JSONObject repo = items.getJSONObject(i);
|
||||
|
||||
String titre = repo.getString("full_name");
|
||||
String url = repo.getString("html_url");
|
||||
String licence = repo.getJSONObject("license").getString("spdx_id");
|
||||
|
||||
resultats.add(new element(titre, url, licence, "github", "code"));
|
||||
}
|
||||
} catch (org.json.JSONException e) {
|
||||
System.out.println("error but continue : " + e.getMessage());
|
||||
}
|
||||
|
||||
return resultats;
|
||||
}
|
||||
}
|
||||
+46
@@ -0,0 +1,46 @@
|
||||
package org.sillyjirafe.uoapi;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import org.sillyjirafe.uoapi.binds.*;
|
||||
|
||||
public class Out {
|
||||
List<Api> Apis = new ArrayList<>();
|
||||
|
||||
public List<String> PreOut()
|
||||
{
|
||||
Apis.add(new GitHubMITApi());
|
||||
Apis.add(new GitHubLGPLApi());
|
||||
Apis.add(new GitHubApacheApi());
|
||||
Apis.add(new OpenVerseImagesApi());
|
||||
Apis.add(new OpenVerseAudiosApi());
|
||||
Apis.add(new PixabayImagesApi());
|
||||
|
||||
List<String> bindnames = new ArrayList<>();
|
||||
|
||||
for (Api api : Apis)
|
||||
{
|
||||
bindnames.add(api.getName());
|
||||
}
|
||||
|
||||
return bindnames;
|
||||
}
|
||||
|
||||
public List<element> out(Map<String, Integer> bindpages, Map<String, String> ApiKeys, Map<String, String> Rnd) throws Exception
|
||||
{
|
||||
List<element> Elements = new ArrayList<>();
|
||||
|
||||
for (int i = 0; i < Apis.size(); i++)
|
||||
{
|
||||
List<element> currentapi = Caller.CallApi(Apis.get(i), bindpages.get(Apis.get(i).getName()), ApiKeys.get(Apis.get(i).getName()), );
|
||||
for (int j = 0; j < currentapi.size(); j++)
|
||||
{
|
||||
Elements.add(currentapi.get(j));
|
||||
}
|
||||
}
|
||||
return Elements;
|
||||
}
|
||||
}
|
||||
+43
@@ -0,0 +1,43 @@
|
||||
package org.sillyjirafe.uoapi.binds;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.json.JSONArray;
|
||||
import org.json.JSONObject;
|
||||
import org.sillyjirafe.uoapi.Api;
|
||||
import org.sillyjirafe.uoapi.element;
|
||||
|
||||
public class GitHubApacheApi implements Api
|
||||
{
|
||||
public String getName()
|
||||
{
|
||||
return "GitHubApacheApi";
|
||||
}
|
||||
|
||||
public String getURL(int page, String apikey, String rnd) {
|
||||
return "https://api.github.com/search/repositories?q="+ rnd +"&license:apache-2.0&page=" + page;
|
||||
}
|
||||
|
||||
public List<element> Parse(String jsonRaw) {
|
||||
List<element> resultats = new ArrayList<>();
|
||||
|
||||
JSONObject json = new JSONObject(jsonRaw);
|
||||
JSONArray items = json.getJSONArray("items");
|
||||
|
||||
for (int i = 0; i < items.length(); i++) {
|
||||
JSONObject repo = items.getJSONObject(i);
|
||||
|
||||
String titre = repo.getString("full_name");
|
||||
String url = repo.getString("html_url");
|
||||
String licence = repo.getJSONObject("license").getString("spdx_id");
|
||||
|
||||
resultats.add(new element(titre, url, licence, "github", "code"));
|
||||
}
|
||||
} catch (org.json.JSONException e) {
|
||||
System.out.println("error but continue : " + e.getMessage());
|
||||
}
|
||||
|
||||
return resultats;
|
||||
}
|
||||
}
|
||||
+54
@@ -0,0 +1,54 @@
|
||||
package org.sillyjirafe.linker;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import org.sillyjirafe.db.Crawl_StateDB;
|
||||
import org.sillyjirafe.db.ElementsDB;
|
||||
import org.sillyjirafe.json.ApiKeys;
|
||||
import org.sillyjirafe.uoapi.*;
|
||||
|
||||
public class Main {
|
||||
public static void main(String[] args) throws Exception
|
||||
{
|
||||
Out Out = new Out();
|
||||
ElementsDB elementDB = new ElementsDB("jdbc:sqlite:elements.db");
|
||||
Crawl_StateDB crawlstateDB = new Crawl_StateDB("jdbc:sqlite:crawlstates.db");
|
||||
elementDB.LinkToDB();
|
||||
crawlstateDB.LinkToDB();
|
||||
Map<String, Boolean> succes = Out.getSucces();
|
||||
|
||||
List<String> bindNames = Out.PreOut();
|
||||
|
||||
Map<String, Integer> pages = new HashMap<>();
|
||||
for (String nom : bindNames) {
|
||||
pages.put(nom, crawlstateDB.getPage(nom));
|
||||
}
|
||||
|
||||
Map<String, String> rnds = new HashMap<>();
|
||||
for (String nom : bindNames) {
|
||||
rnds.put(nom, crawlstateDB.getRND(nom));
|
||||
}
|
||||
|
||||
|
||||
List<element> elements = Out.out(pages, ApiKeys.read(), rnds);
|
||||
|
||||
for (int i = 0; i < elements.size(); i++)
|
||||
{
|
||||
element el = elements.get(i);
|
||||
elementDB.AddEllement(el.title(), el.url(), el.licence(), el.source(), el.type());
|
||||
}
|
||||
|
||||
for (String name : bindNames) {
|
||||
if (Succes.getOrDefault(nom, false)) {
|
||||
crawlstateDB.updatePage(nom, ...); // succès normal, avance
|
||||
} else {
|
||||
crawlstateDB.incrementError(nom); // échec, compte une erreur de plus
|
||||
}
|
||||
}
|
||||
|
||||
elementDB.Close();
|
||||
crawlstateDB.Close();
|
||||
}
|
||||
}
|
||||
+43
@@ -0,0 +1,43 @@
|
||||
package org.sillyjirafe.uoapi.binds;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.json.JSONArray;
|
||||
import org.json.JSONObject;
|
||||
import org.sillyjirafe.uoapi.Api;
|
||||
import org.sillyjirafe.uoapi.element;
|
||||
|
||||
public class OpenVerseAudiosApi implements Api
|
||||
{
|
||||
public String getName()
|
||||
{
|
||||
return "OpenVerseAudiosApi";
|
||||
}
|
||||
|
||||
public String getURL(int page, String apikey) {
|
||||
return "https://api.openverse.org/v1/audio/?format=json&page=" + page;
|
||||
}
|
||||
|
||||
public List<element> Parse(String jsonRaw) {
|
||||
List<element> resultats = new ArrayList<>();
|
||||
|
||||
try {
|
||||
JSONObject json = new JSONObject(jsonRaw);
|
||||
JSONArray items = json.getJSONArray("results");
|
||||
|
||||
for (int i = 0; i < items.length(); i++) {
|
||||
JSONObject audio = items.getJSONObject(i);
|
||||
|
||||
String titre = audio.getString("title");
|
||||
String url = audio.getString("url");
|
||||
String licence = audio.getString("license");
|
||||
|
||||
resultats.add(new element(titre, url, licence, "openverse", "audio"));
|
||||
}
|
||||
} catch (org.json.JSONException e) {
|
||||
System.out.println("error but continue : " + e.getMessage());
|
||||
}
|
||||
return resultats;
|
||||
}
|
||||
}
|
||||
+98
@@ -0,0 +1,98 @@
|
||||
package org.sillyjirafe.db;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.DriverManager;
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
|
||||
public class Crawl_StateDB
|
||||
{
|
||||
String URL = new String();
|
||||
String insertSql = "INSERT OR IGNORE INTO crawlstates (bind_name, last_page, rnd, error) VALUES (?, ?, ?)";
|
||||
Connection connection;
|
||||
|
||||
public Crawl_StateDB(String BDstr) throws SQLException
|
||||
{
|
||||
URL = BDstr;
|
||||
connection = DriverManager.getConnection(URL);
|
||||
}
|
||||
|
||||
public void LinkToDB() throws SQLException
|
||||
{
|
||||
try (Statement stmt = connection.createStatement())
|
||||
{
|
||||
stmt.execute("""
|
||||
CREATE TABLE IF NOT EXISTS crawlstates (
|
||||
bind_name TEXT PRIMARY KEY,
|
||||
last_page INTEGER NOT NULL DEFAULT 1,
|
||||
rnd TEXT NOT NULL DEFAULT 'a'
|
||||
)
|
||||
""");
|
||||
}
|
||||
}
|
||||
|
||||
public void AddEllement(String bind, int lastpage) throws SQLException
|
||||
{
|
||||
try (PreparedStatement ps = connection.prepareStatement(insertSql))
|
||||
{
|
||||
ps.setString(1, bind);
|
||||
ps.setInt(2, lastpage);
|
||||
ps.executeUpdate();
|
||||
System.out.printf("%s is now page %d \n", bind, lastpage);
|
||||
}
|
||||
}
|
||||
|
||||
public int getPage(String bindName) throws SQLException {
|
||||
String sql = "SELECT last_page FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("last_page");
|
||||
} else {
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
public String getRND(String bindName) throws SQLException {
|
||||
String sql = "SELECT rnd FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getString("rnd");
|
||||
} else {
|
||||
return "a";
|
||||
}
|
||||
}
|
||||
}
|
||||
public void updatePage(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET last_page = last_page + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void updateletter(String bindName)
|
||||
{
|
||||
|
||||
}
|
||||
|
||||
public void Close() throws SQLException
|
||||
{
|
||||
if (connection != null)
|
||||
{
|
||||
connection.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
+124
@@ -0,0 +1,124 @@
|
||||
package org.sillyjirafe.db;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.DriverManager;
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
|
||||
public class Crawl_StateDB
|
||||
{
|
||||
String URL = new String();
|
||||
String insertSql = "INSERT OR IGNORE INTO crawlstates (bind_name, last_page, rnd, error) VALUES (?, ?, ?, ?)";
|
||||
Connection connection;
|
||||
|
||||
public Crawl_StateDB(String BDstr) throws SQLException
|
||||
{
|
||||
URL = BDstr;
|
||||
connection = DriverManager.getConnection(URL);
|
||||
}
|
||||
|
||||
public void LinkToDB() throws SQLException
|
||||
{
|
||||
try (Statement stmt = connection.createStatement())
|
||||
{
|
||||
stmt.execute("""
|
||||
CREATE TABLE IF NOT EXISTS crawlstates (
|
||||
bind_name TEXT PRIMARY KEY,
|
||||
last_page INTEGER NOT NULL DEFAULT 1,
|
||||
rnd TEXT NOT NULL DEFAULT 'a'
|
||||
error INTEGER NOT NULL DEFAULT 0,
|
||||
)
|
||||
""");
|
||||
}
|
||||
}
|
||||
|
||||
public void AddEllement(String bind, int lastpage) throws SQLException
|
||||
{
|
||||
try (PreparedStatement ps = connection.prepareStatement(insertSql))
|
||||
{
|
||||
ps.setString(1, bind);
|
||||
ps.setInt(2, lastpage);
|
||||
ps.executeUpdate();
|
||||
System.out.printf("%s is now page %d \n", bind, lastpage);
|
||||
}
|
||||
}
|
||||
|
||||
public int getPage(String bindName) throws SQLException {
|
||||
String sql = "SELECT last_page FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("last_page");
|
||||
} else {
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
public String getRND(String bindName) throws SQLException {
|
||||
String sql = "SELECT rnd FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getString("rnd");
|
||||
} else {
|
||||
return "a";
|
||||
}
|
||||
}
|
||||
}
|
||||
public int getErrors(String bindName) throws SQLException {
|
||||
String sql = "SELECT error FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("error");
|
||||
} else {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
public void updatePage(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, rnd) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET rnd = rnd + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void newError(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET error = error + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void updateletter(String bindName)
|
||||
{
|
||||
|
||||
}
|
||||
|
||||
public void Close() throws SQLException
|
||||
{
|
||||
if (connection != null)
|
||||
{
|
||||
connection.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
+43
@@ -0,0 +1,43 @@
|
||||
package org.sillyjirafe.uoapi.binds;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.json.JSONArray;
|
||||
import org.json.JSONObject;
|
||||
import org.sillyjirafe.uoapi.Api;
|
||||
import org.sillyjirafe.uoapi.element;
|
||||
|
||||
public class OpenVerseImagesApi implements Api
|
||||
{
|
||||
public String getName()
|
||||
{
|
||||
return "OpenVerseImagesApi";
|
||||
}
|
||||
|
||||
public String getURL(int page, String apikey, String rnd) {
|
||||
return "https://api.openverse.org/v1/images/?q=\n" + rnd+"format=json&page=" + page;
|
||||
}
|
||||
|
||||
public List<element> Parse(String jsonRaw) {
|
||||
List<element> resultats = new ArrayList<>();
|
||||
|
||||
try {
|
||||
JSONObject json = new JSONObject(jsonRaw);
|
||||
JSONArray items = json.getJSONArray("results");
|
||||
|
||||
for (int i = 0; i < items.length(); i++) {
|
||||
JSONObject image = items.getJSONObject(i);
|
||||
|
||||
String titre = image.getString("title");
|
||||
String url = image.getString("url");
|
||||
String licence = image.getString("license");
|
||||
|
||||
resultats.add(new element(titre, url, licence, "openverse", "image"));
|
||||
}
|
||||
} catch (org.json.JSONException e) {
|
||||
System.out.println("error but continue : " + e.getMessage());
|
||||
}
|
||||
return resultats;
|
||||
}
|
||||
}
|
||||
+43
@@ -0,0 +1,43 @@
|
||||
package org.sillyjirafe.uoapi.binds;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.json.JSONArray;
|
||||
import org.json.JSONObject;
|
||||
import org.sillyjirafe.uoapi.Api;
|
||||
import org.sillyjirafe.uoapi.element;
|
||||
|
||||
public class OpenVerseImagesApi implements Api
|
||||
{
|
||||
public String getName()
|
||||
{
|
||||
return "OpenVerseImagesApi";
|
||||
}
|
||||
|
||||
public String getURL(int page, String apikey, String rnd) {
|
||||
return "https://api.openverse.org/v1/images/?q=\"" + rnd+"\"format=json&page=" + page;
|
||||
}
|
||||
|
||||
public List<element> Parse(String jsonRaw) {
|
||||
List<element> resultats = new ArrayList<>();
|
||||
|
||||
try {
|
||||
JSONObject json = new JSONObject(jsonRaw);
|
||||
JSONArray items = json.getJSONArray("results");
|
||||
|
||||
for (int i = 0; i < items.length(); i++) {
|
||||
JSONObject image = items.getJSONObject(i);
|
||||
|
||||
String titre = image.getString("title");
|
||||
String url = image.getString("url");
|
||||
String licence = image.getString("license");
|
||||
|
||||
resultats.add(new element(titre, url, licence, "openverse", "image"));
|
||||
}
|
||||
} catch (org.json.JSONException e) {
|
||||
System.out.println("error but continue : " + e.getMessage());
|
||||
}
|
||||
return resultats;
|
||||
}
|
||||
}
|
||||
+45
@@ -0,0 +1,45 @@
|
||||
package org.sillyjirafe.uoapi.binds;
|
||||
|
||||
import org.json.JSONArray;
|
||||
import org.json.JSONObject;
|
||||
import org.sillyjirafe.uoapi.Api;
|
||||
import org.sillyjirafe.uoapi.element;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
public class GitHubMITApi implements Api
|
||||
{
|
||||
|
||||
public String getName()
|
||||
{
|
||||
return "GitHubMITApi";
|
||||
}
|
||||
|
||||
public String getURL(int page, String apikey, String rnd) {
|
||||
return "https://api.github.com/search/repositories?q="+ rnd +"&license:mit&page=" + page;
|
||||
}
|
||||
|
||||
public List<element> Parse(String jsonRaw) {
|
||||
List<element> resultats = new ArrayList<>();
|
||||
|
||||
|
||||
try {
|
||||
JSONObject json = new JSONObject(jsonRaw);
|
||||
JSONArray items = json.getJSONArray("items");
|
||||
|
||||
for (int i = 0; i < items.length(); i++) {
|
||||
JSONObject repo = items.getJSONObject(i);
|
||||
|
||||
String titre = repo.getString("full_name");
|
||||
String url = repo.getString("html_url");
|
||||
String licence = repo.getJSONObject("license").getString("spdx_id");
|
||||
|
||||
resultats.add(new element(titre, url, licence, "github", "code"));
|
||||
}
|
||||
} catch (org.json.JSONException e) {
|
||||
System.out.println("error but continue : " + e.getMessage());
|
||||
}
|
||||
|
||||
return resultats;
|
||||
}
|
||||
}
|
||||
+110
@@ -0,0 +1,110 @@
|
||||
package org.sillyjirafe.db;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.DriverManager;
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
|
||||
public class Crawl_StateDB
|
||||
{
|
||||
String URL = new String();
|
||||
String insertSql = "INSERT OR IGNORE INTO crawlstates (bind_name, last_page, rnd, error) VALUES (?, ?, ?, ?)";
|
||||
Connection connection;
|
||||
|
||||
public Crawl_StateDB(String BDstr) throws SQLException
|
||||
{
|
||||
URL = BDstr;
|
||||
connection = DriverManager.getConnection(URL);
|
||||
}
|
||||
|
||||
public void LinkToDB() throws SQLException
|
||||
{
|
||||
try (Statement stmt = connection.createStatement())
|
||||
{
|
||||
stmt.execute("""
|
||||
CREATE TABLE IF NOT EXISTS crawlstates (
|
||||
bind_name TEXT PRIMARY KEY,
|
||||
last_page INTEGER NOT NULL DEFAULT 1,
|
||||
rnd TEXT NOT NULL DEFAULT 'a'
|
||||
error INTEGER NOT NULL DEFAULT 0,
|
||||
)
|
||||
""");
|
||||
}
|
||||
}
|
||||
|
||||
public void AddEllement(String bind, int lastpage) throws SQLException
|
||||
{
|
||||
try (PreparedStatement ps = connection.prepareStatement(insertSql))
|
||||
{
|
||||
ps.setString(1, bind);
|
||||
ps.setInt(2, lastpage);
|
||||
ps.executeUpdate();
|
||||
System.out.printf("%s is now page %d \n", bind, lastpage);
|
||||
}
|
||||
}
|
||||
|
||||
public int getPage(String bindName) throws SQLException {
|
||||
String sql = "SELECT last_page FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("last_page");
|
||||
} else {
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
public String getRND(String bindName) throws SQLException {
|
||||
String sql = "SELECT rnd FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getString("rnd");
|
||||
} else {
|
||||
return "a";
|
||||
}
|
||||
}
|
||||
}
|
||||
public void updatePage(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, rnd) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET rnd = rnd + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void newError(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET last_page = last_page + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void updateletter(String bindName)
|
||||
{
|
||||
|
||||
}
|
||||
|
||||
public void Close() throws SQLException
|
||||
{
|
||||
if (connection != null)
|
||||
{
|
||||
connection.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
+44
@@ -0,0 +1,44 @@
|
||||
package org.sillyjirafe.uoapi.binds;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.json.JSONArray;
|
||||
import org.json.JSONObject;
|
||||
import org.sillyjirafe.uoapi.Api;
|
||||
import org.sillyjirafe.uoapi.element;
|
||||
|
||||
public class GitHubApacheApi implements Api
|
||||
{
|
||||
public String getName()
|
||||
{
|
||||
return "GitHubApacheApi";
|
||||
}
|
||||
|
||||
public String getURL(int page, String apikey) {
|
||||
return "https://api.github.com/search/repositories?q=license:apache-2.0&page=" + page;
|
||||
}
|
||||
|
||||
public List<element> Parse(String jsonRaw) {
|
||||
List<element> resultats = new ArrayList<>();
|
||||
|
||||
try {
|
||||
JSONObject json = new JSONObject(jsonRaw);
|
||||
JSONArray items = json.getJSONArray("items");
|
||||
|
||||
for (int i = 0; i < items.length(); i++) {
|
||||
JSONObject repo = items.getJSONObject(i);
|
||||
|
||||
String titre = repo.getString("full_name");
|
||||
String url = repo.getString("html_url");
|
||||
String licence = repo.getJSONObject("license").getString("spdx_id");
|
||||
|
||||
resultats.add(new element(titre, url, licence, "github", "code"));
|
||||
}
|
||||
} catch (org.json.JSONException e) {
|
||||
System.out.println("error but continue : " + e.getMessage());
|
||||
}
|
||||
|
||||
return resultats;
|
||||
}
|
||||
}
|
||||
+99
@@ -0,0 +1,99 @@
|
||||
package org.sillyjirafe.db;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.DriverManager;
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
|
||||
public class Crawl_StateDB
|
||||
{
|
||||
String URL = new String();
|
||||
String insertSql = "INSERT OR IGNORE INTO crawlstates (bind_name, last_page, rnd, error) VALUES (?, ?, ?, ?)";
|
||||
Connection connection;
|
||||
|
||||
public Crawl_StateDB(String BDstr) throws SQLException
|
||||
{
|
||||
URL = BDstr;
|
||||
connection = DriverManager.getConnection(URL);
|
||||
}
|
||||
|
||||
public void LinkToDB() throws SQLException
|
||||
{
|
||||
try (Statement stmt = connection.createStatement())
|
||||
{
|
||||
stmt.execute("""
|
||||
CREATE TABLE IF NOT EXISTS crawlstates (
|
||||
bind_name TEXT PRIMARY KEY,
|
||||
last_page INTEGER NOT NULL DEFAULT 1,
|
||||
rnd TEXT NOT NULL DEFAULT 'a'
|
||||
error INTEGER DEFAULT 0,
|
||||
)
|
||||
""");
|
||||
}
|
||||
}
|
||||
|
||||
public void AddEllement(String bind, int lastpage) throws SQLException
|
||||
{
|
||||
try (PreparedStatement ps = connection.prepareStatement(insertSql))
|
||||
{
|
||||
ps.setString(1, bind);
|
||||
ps.setInt(2, lastpage);
|
||||
ps.executeUpdate();
|
||||
System.out.printf("%s is now page %d \n", bind, lastpage);
|
||||
}
|
||||
}
|
||||
|
||||
public int getPage(String bindName) throws SQLException {
|
||||
String sql = "SELECT last_page FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("last_page");
|
||||
} else {
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
public String getRND(String bindName) throws SQLException {
|
||||
String sql = "SELECT rnd FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getString("rnd");
|
||||
} else {
|
||||
return "a";
|
||||
}
|
||||
}
|
||||
}
|
||||
public void updatePage(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET last_page = last_page + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void updateletter(String bindName)
|
||||
{
|
||||
|
||||
}
|
||||
|
||||
public void Close() throws SQLException
|
||||
{
|
||||
if (connection != null)
|
||||
{
|
||||
connection.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
+55
@@ -0,0 +1,55 @@
|
||||
package org.sillyjirafe.linker;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import org.sillyjirafe.db.Crawl_StateDB;
|
||||
import org.sillyjirafe.db.ElementsDB;
|
||||
import org.sillyjirafe.json.ApiKeys;
|
||||
import org.sillyjirafe.uoapi.*;
|
||||
|
||||
public class Main {
|
||||
public static void main(String[] args) throws Exception
|
||||
{
|
||||
Out Out = new Out();
|
||||
ElementsDB elementDB = new ElementsDB("jdbc:sqlite:elements.db");
|
||||
Crawl_StateDB crawlstateDB = new Crawl_StateDB("jdbc:sqlite:crawlstates.db");
|
||||
elementDB.LinkToDB();
|
||||
crawlstateDB.LinkToDB();
|
||||
|
||||
Map<String, Boolean> succes = Out.getSucces();
|
||||
|
||||
List<String> bindNames = Out.PreOut();
|
||||
|
||||
Map<String, Integer> pages = new HashMap<>();
|
||||
for (String name : bindNames) {
|
||||
pages.put(name, crawlstateDB.getPage(name));
|
||||
}
|
||||
|
||||
Map<String, String> rnds = new HashMap<>();
|
||||
for (String nom : bindNames) {
|
||||
rnds.put(nom, crawlstateDB.getRND(nom));
|
||||
}
|
||||
|
||||
|
||||
List<element> elements = Out.out(pages, ApiKeys.read(), rnds);
|
||||
|
||||
for (int i = 0; i < elements.size(); i++)
|
||||
{
|
||||
element el = elements.get(i);
|
||||
elementDB.AddEllement(el.title(), el.url(), el.licence(), el.source(), el.type());
|
||||
}
|
||||
|
||||
for (String name : bindNames) {
|
||||
if (succes.getOrDefault(name, false)) {
|
||||
crawlstateDB.updatePage(name);
|
||||
} else {
|
||||
crawlstateDB.newError(name);
|
||||
}
|
||||
}
|
||||
|
||||
elementDB.Close();
|
||||
crawlstateDB.Close();
|
||||
}
|
||||
}
|
||||
+44
@@ -0,0 +1,44 @@
|
||||
package org.sillyjirafe.uoapi.binds;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.json.JSONArray;
|
||||
import org.json.JSONObject;
|
||||
import org.sillyjirafe.uoapi.Api;
|
||||
import org.sillyjirafe.uoapi.element;
|
||||
|
||||
public class GitHubApacheApi implements Api
|
||||
{
|
||||
public String getName()
|
||||
{
|
||||
return "GitHubApacheApi";
|
||||
}
|
||||
|
||||
public String getURL(int page, String apikey, String rnd) {
|
||||
return "https://api.github.com/search/repositories?q=license:apache-2.0&page=" + page;
|
||||
}
|
||||
|
||||
public List<element> Parse(String jsonRaw) {
|
||||
List<element> resultats = new ArrayList<>();
|
||||
|
||||
try {
|
||||
JSONObject json = new JSONObject(jsonRaw);
|
||||
JSONArray items = json.getJSONArray("items");
|
||||
|
||||
for (int i = 0; i < items.length(); i++) {
|
||||
JSONObject repo = items.getJSONObject(i);
|
||||
|
||||
String titre = repo.getString("full_name");
|
||||
String url = repo.getString("html_url");
|
||||
String licence = repo.getJSONObject("license").getString("spdx_id");
|
||||
|
||||
resultats.add(new element(titre, url, licence, "github", "code"));
|
||||
}
|
||||
} catch (org.json.JSONException e) {
|
||||
System.out.println("error but continue : " + e.getMessage());
|
||||
}
|
||||
|
||||
return resultats;
|
||||
}
|
||||
}
|
||||
+150
@@ -0,0 +1,150 @@
|
||||
package org.sillyjirafe.db;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.DriverManager;
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
|
||||
public class Crawl_StateDB
|
||||
{
|
||||
String URL = new String();
|
||||
String insertSql = "INSERT OR IGNORE INTO crawlstates (bind_name, last_page, rnd, error) VALUES (?, ?, ?, ?)";
|
||||
Connection connection;
|
||||
|
||||
public Crawl_StateDB(String BDstr) throws SQLException
|
||||
{
|
||||
URL = BDstr;
|
||||
connection = DriverManager.getConnection(URL);
|
||||
}
|
||||
|
||||
public void LinkToDB() throws SQLException
|
||||
{
|
||||
try (Statement stmt = connection.createStatement())
|
||||
{
|
||||
stmt.execute("""
|
||||
CREATE TABLE IF NOT EXISTS crawlstates (
|
||||
bind_name TEXT PRIMARY KEY,
|
||||
last_page INTEGER NOT NULL DEFAULT 1,
|
||||
rnd TEXT NOT NULL DEFAULT 'a'
|
||||
error INTEGER NOT NULL DEFAULT 0,
|
||||
)
|
||||
""");
|
||||
}
|
||||
}
|
||||
|
||||
public void AddEllement(String bind, int lastpage) throws SQLException
|
||||
{
|
||||
try (PreparedStatement ps = connection.prepareStatement(insertSql))
|
||||
{
|
||||
ps.setString(1, bind);
|
||||
ps.setInt(2, lastpage);
|
||||
ps.executeUpdate();
|
||||
System.out.printf("%s is now page %d \n", bind, lastpage);
|
||||
}
|
||||
}
|
||||
|
||||
public int getPage(String bindName) throws SQLException {
|
||||
String sql = "SELECT last_page FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("last_page");
|
||||
} else {
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
public String getRND(String bindName) throws SQLException {
|
||||
String sql = "SELECT rnd FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getString("rnd");
|
||||
} else {
|
||||
return "a";
|
||||
}
|
||||
}
|
||||
}
|
||||
public int getErrors(String bindName) throws SQLException {
|
||||
String sql = "SELECT error FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("error");
|
||||
} else {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
public void updatePage(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET last_page = las_page + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void newError(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET error = error + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void resetError(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET error = 0
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void resetPage(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET last_page = 0
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void updateletter(String bindName) throws SQLException
|
||||
{
|
||||
if (getErrors(bindName) >= 10)
|
||||
{
|
||||
resetError(bindName);
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
public void Close() throws SQLException
|
||||
{
|
||||
if (connection != null)
|
||||
{
|
||||
connection.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
+124
@@ -0,0 +1,124 @@
|
||||
package org.sillyjirafe.db;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.DriverManager;
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
|
||||
public class Crawl_StateDB
|
||||
{
|
||||
String URL = new String();
|
||||
String insertSql = "INSERT OR IGNORE INTO crawlstates (bind_name, last_page, rnd, error) VALUES (?, ?, ?, ?)";
|
||||
Connection connection;
|
||||
|
||||
public Crawl_StateDB(String BDstr) throws SQLException
|
||||
{
|
||||
URL = BDstr;
|
||||
connection = DriverManager.getConnection(URL);
|
||||
}
|
||||
|
||||
public void LinkToDB() throws SQLException
|
||||
{
|
||||
try (Statement stmt = connection.createStatement())
|
||||
{
|
||||
stmt.execute("""
|
||||
CREATE TABLE IF NOT EXISTS crawlstates (
|
||||
bind_name TEXT PRIMARY KEY,
|
||||
last_page INTEGER NOT NULL DEFAULT 1,
|
||||
rnd TEXT NOT NULL DEFAULT 'a'
|
||||
error INTEGER NOT NULL DEFAULT 0,
|
||||
)
|
||||
""");
|
||||
}
|
||||
}
|
||||
|
||||
public void AddEllement(String bind, int lastpage) throws SQLException
|
||||
{
|
||||
try (PreparedStatement ps = connection.prepareStatement(insertSql))
|
||||
{
|
||||
ps.setString(1, bind);
|
||||
ps.setInt(2, lastpage);
|
||||
ps.executeUpdate();
|
||||
System.out.printf("%s is now page %d \n", bind, lastpage);
|
||||
}
|
||||
}
|
||||
|
||||
public int getPage(String bindName) throws SQLException {
|
||||
String sql = "SELECT last_page FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("last_page");
|
||||
} else {
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
public String getRND(String bindName) throws SQLException {
|
||||
String sql = "SELECT rnd FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getString("rnd");
|
||||
} else {
|
||||
return "a";
|
||||
}
|
||||
}
|
||||
}
|
||||
public int getErrors(String bindName) throws SQLException {
|
||||
String sql = "SELECT error FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("error");
|
||||
} else {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
public void updatePage(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, rnd) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET rnd = rnd + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void newError(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET last_page = last_page + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void updateletter(String bindName)
|
||||
{
|
||||
|
||||
}
|
||||
|
||||
public void Close() throws SQLException
|
||||
{
|
||||
if (connection != null)
|
||||
{
|
||||
connection.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
+150
@@ -0,0 +1,150 @@
|
||||
package org.sillyjirafe.db;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.DriverManager;
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
|
||||
public class Crawl_StateDB
|
||||
{
|
||||
String URL = new String();
|
||||
String insertSql = "INSERT OR IGNORE INTO crawlstates (bind_name, last_page, rnd, error) VALUES (?, ?, ?, ?)";
|
||||
Connection connection;
|
||||
|
||||
public Crawl_StateDB(String BDstr) throws SQLException
|
||||
{
|
||||
URL = BDstr;
|
||||
connection = DriverManager.getConnection(URL);
|
||||
}
|
||||
|
||||
public void LinkToDB() throws SQLException
|
||||
{
|
||||
try (Statement stmt = connection.createStatement())
|
||||
{
|
||||
stmt.execute("""
|
||||
CREATE TABLE IF NOT EXISTS crawlstates (
|
||||
bind_name TEXT PRIMARY KEY,
|
||||
last_page INTEGER NOT NULL DEFAULT 1,
|
||||
rnd TEXT NOT NULL DEFAULT 'a'
|
||||
error INTEGER NOT NULL DEFAULT 0,
|
||||
)
|
||||
""");
|
||||
}
|
||||
}
|
||||
|
||||
public void AddEllement(String bind, int lastpage) throws SQLException
|
||||
{
|
||||
try (PreparedStatement ps = connection.prepareStatement(insertSql))
|
||||
{
|
||||
ps.setString(1, bind);
|
||||
ps.setInt(2, lastpage);
|
||||
ps.executeUpdate();
|
||||
System.out.printf("%s is now page %d \n", bind, lastpage);
|
||||
}
|
||||
}
|
||||
|
||||
public int getPage(String bindName) throws SQLException {
|
||||
String sql = "SELECT last_page FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("last_page");
|
||||
} else {
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
public String getRND(String bindName) throws SQLException {
|
||||
String sql = "SELECT rnd FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getString("rnd");
|
||||
} else {
|
||||
return "a";
|
||||
}
|
||||
}
|
||||
}
|
||||
public int getErrors(String bindName) throws SQLException {
|
||||
String sql = "SELECT error FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("error");
|
||||
} else {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
public void updatePage(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET last_page = las_page + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void newError(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET error = error + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void resetError(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET error = 0
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void resetPage(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET last_page = 0
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void updateletter(String bindName) throws SQLException
|
||||
{
|
||||
if (getErrors(bindName) >= 10)
|
||||
{
|
||||
resetError(bindName);
|
||||
resetPage(bindName);
|
||||
}
|
||||
}
|
||||
|
||||
public void Close() throws SQLException
|
||||
{
|
||||
if (connection != null)
|
||||
{
|
||||
connection.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
+43
@@ -0,0 +1,43 @@
|
||||
package org.sillyjirafe.uoapi.binds;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.json.JSONArray;
|
||||
import org.json.JSONObject;
|
||||
import org.sillyjirafe.uoapi.Api;
|
||||
import org.sillyjirafe.uoapi.element;
|
||||
|
||||
public class OpenVerseAudiosApi implements Api
|
||||
{
|
||||
public String getName()
|
||||
{
|
||||
return "OpenVerseAudiosApi";
|
||||
}
|
||||
|
||||
public String getURL(int page, String apikey, String rnd) {
|
||||
return "https://api.openverse.org/v1/audio/?q="+ rnd +"format=json&page=" + page;
|
||||
}
|
||||
|
||||
public List<element> Parse(String jsonRaw) {
|
||||
List<element> resultats = new ArrayList<>();
|
||||
|
||||
try {
|
||||
JSONObject json = new JSONObject(jsonRaw);
|
||||
JSONArray items = json.getJSONArray("results");
|
||||
|
||||
for (int i = 0; i < items.length(); i++) {
|
||||
JSONObject audio = items.getJSONObject(i);
|
||||
|
||||
String titre = audio.getString("title");
|
||||
String url = audio.getString("url");
|
||||
String licence = audio.getString("license");
|
||||
|
||||
resultats.add(new element(titre, url, licence, "openverse", "audio"));
|
||||
}
|
||||
} catch (org.json.JSONException e) {
|
||||
System.out.println("error but continue : " + e.getMessage());
|
||||
}
|
||||
return resultats;
|
||||
}
|
||||
}
|
||||
+139
@@ -0,0 +1,139 @@
|
||||
package org.sillyjirafe.db;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.DriverManager;
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
|
||||
public class Crawl_StateDB
|
||||
{
|
||||
String URL = new String();
|
||||
String insertSql = "INSERT OR IGNORE INTO crawlstates (bind_name, last_page, rnd, error) VALUES (?, ?, ?, ?)";
|
||||
Connection connection;
|
||||
|
||||
public Crawl_StateDB(String BDstr) throws SQLException
|
||||
{
|
||||
URL = BDstr;
|
||||
connection = DriverManager.getConnection(URL);
|
||||
}
|
||||
|
||||
public void LinkToDB() throws SQLException
|
||||
{
|
||||
try (Statement stmt = connection.createStatement())
|
||||
{
|
||||
stmt.execute("""
|
||||
CREATE TABLE IF NOT EXISTS crawlstates (
|
||||
bind_name TEXT PRIMARY KEY,
|
||||
last_page INTEGER NOT NULL DEFAULT 1,
|
||||
rnd TEXT NOT NULL DEFAULT 'a'
|
||||
error INTEGER NOT NULL DEFAULT 0,
|
||||
)
|
||||
""");
|
||||
}
|
||||
}
|
||||
|
||||
public void AddEllement(String bind, int lastpage) throws SQLException
|
||||
{
|
||||
try (PreparedStatement ps = connection.prepareStatement(insertSql))
|
||||
{
|
||||
ps.setString(1, bind);
|
||||
ps.setInt(2, lastpage);
|
||||
ps.executeUpdate();
|
||||
System.out.printf("%s is now page %d \n", bind, lastpage);
|
||||
}
|
||||
}
|
||||
|
||||
public int getPage(String bindName) throws SQLException {
|
||||
String sql = "SELECT last_page FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("last_page");
|
||||
} else {
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
public String getRND(String bindName) throws SQLException {
|
||||
String sql = "SELECT rnd FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getString("rnd");
|
||||
} else {
|
||||
return "a";
|
||||
}
|
||||
}
|
||||
}
|
||||
public int getErrors(String bindName) throws SQLException {
|
||||
String sql = "SELECT error FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("error");
|
||||
} else {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
public void updatePage(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET last_page = las_page + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void newError(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET error = error + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void resetError(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET error = 0
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void updateletter(String bindName) throws SQLException
|
||||
{
|
||||
if (getErrors(bindName) >= 10)
|
||||
{
|
||||
resetError(bindName);
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
public void Close() throws SQLException
|
||||
{
|
||||
if (connection != null)
|
||||
{
|
||||
connection.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
+44
@@ -0,0 +1,44 @@
|
||||
package org.sillyjirafe.uoapi.binds;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.json.JSONArray;
|
||||
import org.json.JSONObject;
|
||||
import org.sillyjirafe.uoapi.Api;
|
||||
import org.sillyjirafe.uoapi.element;
|
||||
|
||||
public class PixabayImagesApi implements Api
|
||||
{
|
||||
public String getName()
|
||||
{
|
||||
return "PixabayImagesApi";
|
||||
}
|
||||
|
||||
public String getURL(int page, String apikey) {
|
||||
return "https://pixabay.com/api/?per_page=200&page=" + page + "&key=" + apikey;
|
||||
}
|
||||
|
||||
public List<element> Parse(String jsonRaw) {
|
||||
List<element> resultats = new ArrayList<>();
|
||||
|
||||
try {
|
||||
JSONObject json = new JSONObject(jsonRaw);
|
||||
JSONArray items = json.getJSONArray("hits");
|
||||
|
||||
for (int i = 0; i < items.length(); i++) {
|
||||
JSONObject repo = items.getJSONObject(i);
|
||||
|
||||
String titre = repo.getString("name");
|
||||
String url = repo.getString("pageURL");
|
||||
String licence = "cc0";
|
||||
|
||||
resultats.add(new element(titre, url, licence, "pixabay", "image"));
|
||||
}
|
||||
} catch (org.json.JSONException e) {
|
||||
System.out.println("error but continue : " + e.getMessage());
|
||||
}
|
||||
|
||||
return resultats;
|
||||
}
|
||||
}
|
||||
+55
@@ -0,0 +1,55 @@
|
||||
package org.sillyjirafe.linker;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import org.sillyjirafe.db.Crawl_StateDB;
|
||||
import org.sillyjirafe.db.ElementsDB;
|
||||
import org.sillyjirafe.json.ApiKeys;
|
||||
import org.sillyjirafe.uoapi.*;
|
||||
|
||||
public class Main {
|
||||
public static void main(String[] args) throws Exception
|
||||
{
|
||||
Out Out = new Out();
|
||||
ElementsDB elementDB = new ElementsDB("jdbc:sqlite:elements.db");
|
||||
Crawl_StateDB crawlstateDB = new Crawl_StateDB("jdbc:sqlite:crawlstates.db");
|
||||
elementDB.LinkToDB();
|
||||
crawlstateDB.LinkToDB();
|
||||
|
||||
Map<String, Boolean> succes = Out.getSucces();
|
||||
|
||||
List<String> bindNames = Out.PreOut();
|
||||
|
||||
Map<String, Integer> pages = new HashMap<>();
|
||||
for (String name : bindNames) {
|
||||
pages.put(name, crawlstateDB.getPage(name));
|
||||
}
|
||||
|
||||
Map<String, String> rnds = new HashMap<>();
|
||||
for (String nom : bindNames) {
|
||||
rnds.put(nom, crawlstateDB.getRND(nom));
|
||||
}
|
||||
|
||||
|
||||
List<element> elements = Out.out(pages, ApiKeys.read(), rnds);
|
||||
|
||||
for (int i = 0; i < elements.size(); i++)
|
||||
{
|
||||
element el = elements.get(i);
|
||||
elementDB.AddEllement(el.title(), el.url(), el.licence(), el.source(), el.type());
|
||||
}
|
||||
|
||||
for (String name : bindNames) {
|
||||
if (succes.getOrDefault(name, false)) {
|
||||
crawlstateDB.updatePage(name);
|
||||
} else {
|
||||
crawlstateDB.incrementError(name); // échec, compte une erreur de plus
|
||||
}
|
||||
}
|
||||
|
||||
elementDB.Close();
|
||||
crawlstateDB.Close();
|
||||
}
|
||||
}
|
||||
+124
@@ -0,0 +1,124 @@
|
||||
package org.sillyjirafe.db;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.DriverManager;
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
|
||||
public class Crawl_StateDB
|
||||
{
|
||||
String URL = new String();
|
||||
String insertSql = "INSERT OR IGNORE INTO crawlstates (bind_name, last_page, rnd, error) VALUES (?, ?, ?, ?)";
|
||||
Connection connection;
|
||||
|
||||
public Crawl_StateDB(String BDstr) throws SQLException
|
||||
{
|
||||
URL = BDstr;
|
||||
connection = DriverManager.getConnection(URL);
|
||||
}
|
||||
|
||||
public void LinkToDB() throws SQLException
|
||||
{
|
||||
try (Statement stmt = connection.createStatement())
|
||||
{
|
||||
stmt.execute("""
|
||||
CREATE TABLE IF NOT EXISTS crawlstates (
|
||||
bind_name TEXT PRIMARY KEY,
|
||||
last_page INTEGER NOT NULL DEFAULT 1,
|
||||
rnd TEXT NOT NULL DEFAULT 'a'
|
||||
error INTEGER NOT NULL DEFAULT 0,
|
||||
)
|
||||
""");
|
||||
}
|
||||
}
|
||||
|
||||
public void AddEllement(String bind, int lastpage) throws SQLException
|
||||
{
|
||||
try (PreparedStatement ps = connection.prepareStatement(insertSql))
|
||||
{
|
||||
ps.setString(1, bind);
|
||||
ps.setInt(2, lastpage);
|
||||
ps.executeUpdate();
|
||||
System.out.printf("%s is now page %d \n", bind, lastpage);
|
||||
}
|
||||
}
|
||||
|
||||
public int getPage(String bindName) throws SQLException {
|
||||
String sql = "SELECT last_page FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("last_page");
|
||||
} else {
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
public String getRND(String bindName) throws SQLException {
|
||||
String sql = "SELECT rnd FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getString("rnd");
|
||||
} else {
|
||||
return "a";
|
||||
}
|
||||
}
|
||||
}
|
||||
public int getErrors(String bindName) throws SQLException {
|
||||
String sql = "SELECT error FROM crawlstates WHERE bind_name = ?";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
|
||||
if (rs.next()) {
|
||||
return rs.getInt("error");
|
||||
} else {
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
public void updatePage(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, rnd) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET rnd = rnd + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void newError(String bindName) throws SQLException {
|
||||
String sql = """
|
||||
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
|
||||
ON CONFLICT(bind_name) DO UPDATE SET last_page = last_page + 1
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, bindName);
|
||||
ps.executeUpdate();
|
||||
}
|
||||
}
|
||||
public void updateletter(String bindName)
|
||||
{
|
||||
|
||||
}
|
||||
|
||||
public void Close() throws SQLException
|
||||
{
|
||||
if (connection != null)
|
||||
{
|
||||
connection.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
+46
@@ -0,0 +1,46 @@
|
||||
package org.sillyjirafe.uoapi;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import org.sillyjirafe.uoapi.binds.*;
|
||||
|
||||
public class Out {
|
||||
List<Api> Apis = new ArrayList<>();
|
||||
|
||||
public List<String> PreOut()
|
||||
{
|
||||
Apis.add(new GitHubMITApi());
|
||||
Apis.add(new GitHubLGPLApi());
|
||||
Apis.add(new GitHubApacheApi());
|
||||
Apis.add(new OpenVerseImagesApi());
|
||||
Apis.add(new OpenVerseAudiosApi());
|
||||
Apis.add(new PixabayImagesApi());
|
||||
|
||||
List<String> bindnames = new ArrayList<>();
|
||||
|
||||
for (Api api : Apis)
|
||||
{
|
||||
bindnames.add(api.getName());
|
||||
}
|
||||
|
||||
return bindnames;
|
||||
}
|
||||
|
||||
public List<element> out(Map<String, Integer> bindpages, Map<String, String> ApiKeys, Map<String, String> Rnds) throws Exception
|
||||
{
|
||||
List<element> Elements = new ArrayList<>();
|
||||
|
||||
for (int i = 0; i < Apis.size(); i++)
|
||||
{
|
||||
List<element> currentapi = Caller.CallApi(Apis.get(i), bindpages.get(Apis.get(i).getName()), ApiKeys.get(Apis.get(i).getName()), Rnds.get(Apis.get(i).getName()));
|
||||
for (int j = 0; j < currentapi.size(); j++)
|
||||
{
|
||||
Elements.add(currentapi.get(j));
|
||||
}
|
||||
}
|
||||
return Elements;
|
||||
}
|
||||
}
|
||||
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
Binary file not shown.
BIN
Binary file not shown.
Binary file not shown.
BIN
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
+1
-1
@@ -1,2 +1,2 @@
|
||||
eclipse.preferences.version=1
|
||||
org.eclipse.jdt.launching.PREF_VM_XML=<?xml version\="1.0" encoding\="UTF-8" standalone\="no"?>\n<vmSettings defaultVM\="57,org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType13,1785979565653" defaultVMConnector\="">\n <vmType id\="org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType">\n <vm id\="1785979565653" name\="jre" path\="/app/eclipse/plugins/org.eclipse.justj.openjdk.hotspot.jre.full.linux.x86_64_21.0.11.v20260515-1531/jre"/>\n </vmType>\n</vmSettings>\n
|
||||
org.eclipse.jdt.launching.PREF_VM_XML=<?xml version\="1.0" encoding\="UTF-8" standalone\="no"?>\n<vmSettings defaultVM\="57,org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType13,1791126493219" defaultVMConnector\="">\n <vmType id\="org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType">\n <vm id\="1791126493219" name\="jre" path\="/app/eclipse/plugins/org.eclipse.justj.openjdk.hotspot.jre.full.linux.x86_64_25.0.4.v20260826-0822/jre"/>\n </vmType>\n</vmSettings>\n
|
||||
@@ -0,0 +1,2 @@
|
||||
eclipse.preferences.version=1
|
||||
org.eclipse.ui.console.zoomFontHeights=org.eclipse.debug.ui.ProcessConsoleType\=10|0
|
||||
@@ -1,6 +1,6 @@
|
||||
IMPORT_FILES_AND_FOLDERS_RELATIVE=true
|
||||
IMPORT_FILES_AND_FOLDERS_TYPE=23,1
|
||||
eclipse.preferences.version=1
|
||||
platformState=945730117414574
|
||||
platformState=943949841082661
|
||||
quickStart=false
|
||||
tipsAndTricks=true
|
||||
@@ -6,6 +6,7 @@
|
||||
<listAttribute key="org.eclipse.debug.core.MAPPED_RESOURCE_TYPES">
|
||||
<listEntry value="1"/>
|
||||
</listAttribute>
|
||||
<stringAttribute key="org.eclipse.debug.core.terminate.timestamp" value="1791670329354"/>
|
||||
<booleanAttribute key="org.eclipse.jdt.launching.ATTR_EXCLUDE_TEST_CODE" value="true"/>
|
||||
<stringAttribute key="org.eclipse.jdt.launching.MAIN_TYPE" value="org.sillyjirafe.linker.Main"/>
|
||||
<stringAttribute key="org.eclipse.jdt.launching.MODULE_NAME" value="SillyBlahaj.org"/>
|
||||
|
||||
@@ -10,16 +10,16 @@
|
||||
<mruHistory/>
|
||||
<favorites/>
|
||||
</launchGroup>
|
||||
<launchGroup id="org.eclipse.ui.externaltools.launchGroup">
|
||||
<mruHistory/>
|
||||
<favorites/>
|
||||
</launchGroup>
|
||||
<launchGroup id="org.eclipse.eclemma.ui.launchGroup.coverage">
|
||||
<mruHistory>
|
||||
<launch memento="<?xml version="1.0" encoding="UTF-8" standalone="no"?> <launchConfiguration local="true" path="Main (3)"/> "/>
|
||||
</mruHistory>
|
||||
<favorites/>
|
||||
</launchGroup>
|
||||
<launchGroup id="org.eclipse.ui.externaltools.launchGroup">
|
||||
<mruHistory/>
|
||||
<favorites/>
|
||||
</launchGroup>
|
||||
<launchGroup id="org.eclipse.debug.ui.launchGroup.run">
|
||||
<mruHistory>
|
||||
<launch memento="<?xml version="1.0" encoding="UTF-8" standalone="no"?> <launchConfiguration local="true" path="Main (3)"/> "/>
|
||||
|
||||
@@ -0,0 +1,3 @@
|
||||
#Eclipse last used NL locale
|
||||
#Sat Oct 10 23:02:27 CEST 2026
|
||||
eclipse.last.nl=en_US
|
||||
File diff suppressed because it is too large.
Load diff
Binary file not shown.
Binary file not shown.
File renamed without changes.
Binary file not shown.
BIN
Binary file not shown.
Binary file not shown.
File renamed without changes.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -1,7 +1,9 @@
|
||||
INDEX VERSION 1.134+/home/kotama/Silly-Jirafe/.metadata/.plugins/org.eclipse.jdt.core
|
||||
1568060524.index
|
||||
4203645977.index
|
||||
1865797976.index
|
||||
206920101.index
|
||||
INDEX VERSION 1.134+/home/Kotama_Chio/Musique/SillyBlahaj.org/Java/.metadata/.plugins/org.eclipse.jdt.core
|
||||
263131634.index
|
||||
32071656.index
|
||||
2407587206.index
|
||||
2916335905.index
|
||||
3079814413.index
|
||||
3263143359.index
|
||||
206920101.index
|
||||
2910310629.index
|
||||
1865797976.index
|
||||
Binary file not shown.
@@ -2,4 +2,6 @@
|
||||
<dirs>
|
||||
<entry loc="/app/eclipse/plugins/org.eclipse.justj.openjdk.hotspot.jre.full.linux.x86_64_21.0.11.v20260515-1531/jre" stamp="0"/>
|
||||
<entry loc="/app/eclipse/plugins/org.eclipse.justj.openjdk.hotspot.jre.full.linux.x86_64_21.0.11.v20260515-1531" stamp="0"/>
|
||||
<entry loc="/app/eclipse/plugins/org.eclipse.justj.openjdk.hotspot.jre.full.linux.x86_64_25.0.4.v20260826-0822" stamp="0"/>
|
||||
<entry loc="/app/eclipse/plugins/org.eclipse.justj.openjdk.hotspot.jre.full.linux.x86_64_25.0.4.v20260826-0822/jre" stamp="0"/>
|
||||
</dirs>
|
||||
@@ -12,4 +12,16 @@
|
||||
<entry path="null"/>
|
||||
</endorsedDirs>
|
||||
</libraryInfo>
|
||||
<libraryInfo home="/app/eclipse/plugins/org.eclipse.justj.openjdk.hotspot.jre.full.linux.x86_64_25.0.4.v20260826-0822" version="25.0.4.1">
|
||||
<bootpath>
|
||||
<entry path="null"/>
|
||||
</bootpath>
|
||||
<extensionDirs>
|
||||
<entry path="null"/>
|
||||
</extensionDirs>
|
||||
<endorsedDirs>
|
||||
<entry path="null"/>
|
||||
</endorsedDirs>
|
||||
</libraryInfo>
|
||||
<libraryInfo home="/app/eclipse/plugins/org.eclipse.justj.openjdk.hotspot.jre.full.linux.x86_64_25.0.4.v20260826-0822/jre" version="25.0.4.1"/>
|
||||
</libraryInfos>
|
||||
@@ -2,13 +2,13 @@
|
||||
<section name="Workbench">
|
||||
<item key="org.eclipse.jdt.ui.last.selected.jre.kind2" value="2"/>
|
||||
<item key="org.eclipse.jdt.ui.last.selected.execution.enviroment" value="JavaSE-21"/>
|
||||
<item key="org.eclipse.jdt.ui.lastextjar" value="/home/kotama/Silly-Jirafe/libs"/>
|
||||
<item key="org.eclipse.jdt.ui.lastextjar" value="/home/Kotama_Chio/Musique/SillyBlahaj.org/Java/libs"/>
|
||||
<section name="org.eclipse.jdt.internal.ui.packageview.PackageExplorerPart">
|
||||
<item key="group_libraries" value="true"/>
|
||||
<item key="layout" value="2"/>
|
||||
<item key="rootMode" value="1"/>
|
||||
<item key="linkWithEditor" value="false"/>
|
||||
<item key="memento" value="<?xml version="1.0" encoding="UTF-8"?>
<packageExplorer group_libraries="1" layout="2" linkWithEditor="0" rootMode="1" workingSetName="Aggregate for window 1785979556746">
<customFilters userDefinedPatternsEnabled="false">
<xmlDefinedFilters>
<child filterId="org.eclipse.jdt.ui.PackageExplorer.StaticsFilter" isEnabled="false"/>
<child filterId="org.eclipse.buildship.ui.packageexplorer.filter.gradle.buildfolder" isEnabled="true"/>
<child filterId="org.eclipse.mylyn.java.ui.MembersFilter" isEnabled="false"/>
<child filterId="org.eclipse.jdt.ui.PackageExplorer.NonJavaProjectsFilter" isEnabled="false"/>
<child filterId="org.eclipse.jdt.ui.PackageExplorer_patternFilterId_.*" isEnabled="false"/>
<child filterId="org.eclipse.jdt.ui.PackageExplorer.NonSharedProjectsFilter" isEnabled="false"/>
<child filterId="org.eclipse.jdt.ui.PackageExplorer.SyntheticMembersFilter" isEnabled="true"/>
<child filterId="org.eclipse.jdt.ui.PackageExplorer.ContainedLibraryFilter" isEnabled="false"/>
<child filterId="org.eclipse.jdt.internal.ui.PackageExplorer.HideInnerClassFilesFilter" isEnabled="true"/>
<child filterId="org.eclipse.jdt.internal.ui.PackageExplorer.EmptyInnerPackageFilter" isEnabled="true"/>
<child filterId="org.eclipse.m2e.MavenModuleFilter" isEnabled="false"/>
<child filterId="org.eclipse.buildship.ui.packageexplorer.filter.gradle.subProject" isEnabled="true"/>
<child filterId="org.eclipse.jdt.ui.PackageExplorer.ClosedProjectsFilter" isEnabled="false"/>
<child filterId="org.eclipse.jdt.ui.PackageExplorer.DeprecatedMembersFilter" isEnabled="false"/>
<child filterId="org.eclipse.jdt.ui.PackageExplorer.EmptyLibraryContainerFilter" isEnabled="true"/>
<child filterId="org.eclipse.jdt.ui.PackageExplorer.PackageDeclarationFilter" isEnabled="true"/>
<child filterId="org.eclipse.jdt.ui.PackageExplorer.ImportDeclarationFilter" isEnabled="true"/>
<child filterId="org.eclipse.jdt.ui.PackageExplorer.NonJavaElementFilter" isEnabled="false"/>
<child filterId="org.eclipse.jdt.ui.PackageExplorer.LibraryFilter" isEnabled="false"/>
<child filterId="org.eclipse.jdt.ui.PackageExplorer.CuAndClassFileFilter" isEnabled="false"/>
<child filterId="org.eclipse.jdt.internal.ui.PackageExplorer.EmptyPackageFilter" isEnabled="false"/>
<child filterId="org.eclipse.jdt.ui.PackageExplorer.NonPublicFilter" isEnabled="false"/>
<child filterId="org.eclipse.jdt.ui.PackageExplorer.LocalTypesFilter" isEnabled="false"/>
<child filterId="org.eclipse.jdt.ui.PackageExplorer.FieldsFilter" isEnabled="false"/>
</xmlDefinedFilters>
</customFilters>
</packageExplorer>"/>
|
||||
<item key="memento" value="<?xml version="1.0" encoding="UTF-8"?>
<packageExplorer group_libraries="1" layout="2" linkWithEditor="0" rootMode="1" workingSetName="Aggregate for window 1785979556746">
<customFilters userDefinedPatternsEnabled="false">
<xmlDefinedFilters>
<child filterId="org.eclipse.jdt.ui.PackageExplorer.StaticsFilter" isEnabled="false"/>
<child filterId="org.eclipse.buildship.ui.packageexplorer.filter.gradle.buildfolder" isEnabled="true"/>
<child filterId="org.eclipse.mylyn.java.ui.MembersFilter" isEnabled="false"/>
<child filterId="org.eclipse.jdt.ui.PackageExplorer.NonJavaProjectsFilter" isEnabled="false"/>
<child filterId="org.eclipse.jdt.ui.PackageExplorer_patternFilterId_.*" isEnabled="false"/>
<child filterId="org.eclipse.jdt.ui.PackageExplorer.NonSharedProjectsFilter" isEnabled="false"/>
<child filterId="org.eclipse.jdt.ui.PackageExplorer.SyntheticMembersFilter" isEnabled="true"/>
<child filterId="org.eclipse.jdt.ui.PackageExplorer.ContainedLibraryFilter" isEnabled="false"/>
<child filterId="org.eclipse.m2e.MavenModuleFilter" isEnabled="false"/>
<child filterId="org.eclipse.jdt.internal.ui.PackageExplorer.HideInnerClassFilesFilter" isEnabled="true"/>
<child filterId="org.eclipse.jdt.internal.ui.PackageExplorer.EmptyInnerPackageFilter" isEnabled="true"/>
<child filterId="org.eclipse.buildship.ui.packageexplorer.filter.gradle.subProject" isEnabled="true"/>
<child filterId="org.eclipse.jdt.ui.PackageExplorer.ClosedProjectsFilter" isEnabled="false"/>
<child filterId="org.eclipse.jdt.ui.PackageExplorer.DeprecatedMembersFilter" isEnabled="false"/>
<child filterId="org.eclipse.jdt.ui.PackageExplorer.EmptyLibraryContainerFilter" isEnabled="true"/>
<child filterId="org.eclipse.jdt.ui.PackageExplorer.PackageDeclarationFilter" isEnabled="true"/>
<child filterId="org.eclipse.jdt.ui.PackageExplorer.ImportDeclarationFilter" isEnabled="true"/>
<child filterId="org.eclipse.jdt.ui.PackageExplorer.NonJavaElementFilter" isEnabled="false"/>
<child filterId="org.eclipse.jdt.ui.PackageExplorer.LibraryFilter" isEnabled="false"/>
<child filterId="org.eclipse.jdt.ui.PackageExplorer.CuAndClassFileFilter" isEnabled="false"/>
<child filterId="org.eclipse.jdt.internal.ui.PackageExplorer.EmptyPackageFilter" isEnabled="false"/>
<child filterId="org.eclipse.jdt.ui.PackageExplorer.NonPublicFilter" isEnabled="false"/>
<child filterId="org.eclipse.jdt.ui.PackageExplorer.LocalTypesFilter" isEnabled="false"/>
<child filterId="org.eclipse.jdt.ui.PackageExplorer.FieldsFilter" isEnabled="false"/>
</xmlDefinedFilters>
</customFilters>
</packageExplorer>"/>
|
||||
</section>
|
||||
<section name="JavaProjectWizard.dialogBounds">
|
||||
<item key="DIALOG_X_ORIGIN" value="0"/>
|
||||
|
||||
+4
@@ -0,0 +1,4 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<session version="1.0">
|
||||
<refactoring comment="Copy element 'libs' to 'sillyjirafe.org.api'
- Original project: 'SillyJirafe.org'
- Destination element: 'sillyjirafe.org.api'
- Original element: 'libs'" description="Copy folder" destination="=sillyjirafe.org.api" element1="libs" files="0" flags="589830" folders="1" id="org.eclipse.jdt.ui.copy" policy="org.eclipse.jdt.ui.copyResources" stamp="1791564728729" units="0" version="1.0"/>
|
||||
</session>
|
||||
+1
@@ -0,0 +1 @@
|
||||
1791564728729 Copy folder
|
||||
@@ -39,3 +39,11 @@
|
||||
2026-09-22 16:37:19,751 [Worker-7: Loading available Gradle versions] INFO o.e.b.c.i.u.g.PublishedGradleVersions - Gradle version information cache is out-of-date. Trying to update.
|
||||
2026-09-25 09:23:41,947 [Worker-6: Loading available Gradle versions] INFO o.e.b.c.i.u.g.PublishedGradleVersions - Gradle version information cache is up-to-date. Trying to read.
|
||||
2026-09-29 23:22:34,391 [Worker-6: Loading available Gradle versions] INFO o.e.b.c.i.u.g.PublishedGradleVersions - Gradle version information cache is up-to-date. Trying to read.
|
||||
2026-10-04 17:08:14,297 [Worker-1: Loading available Gradle versions] INFO o.e.b.c.i.u.g.PublishedGradleVersions - Gradle version information cache is up-to-date. Trying to read.
|
||||
2026-10-04 22:27:35,351 [Worker-1: Loading available Gradle versions] INFO o.e.b.c.i.u.g.PublishedGradleVersions - Gradle version information cache is up-to-date. Trying to read.
|
||||
2026-10-06 13:16:34,293 [Worker-5: Loading available Gradle versions] INFO o.e.b.c.i.u.g.PublishedGradleVersions - Gradle version information cache is out-of-date. Trying to update.
|
||||
2026-10-07 00:08:16,911 [Worker-8: Loading available Gradle versions] INFO o.e.b.c.i.u.g.PublishedGradleVersions - Gradle version information cache is up-to-date. Trying to read.
|
||||
2026-10-07 16:37:42,461 [Worker-2: Loading available Gradle versions] INFO o.e.b.c.i.u.g.PublishedGradleVersions - Gradle version information cache is up-to-date. Trying to read.
|
||||
2026-10-07 21:50:08,486 [Worker-7: Loading available Gradle versions] INFO o.e.b.c.i.u.g.PublishedGradleVersions - Gradle version information cache is up-to-date. Trying to read.
|
||||
2026-10-09 18:50:09,616 [Worker-9: Loading available Gradle versions] INFO o.e.b.c.i.u.g.PublishedGradleVersions - Gradle version information cache is up-to-date. Trying to read.
|
||||
2026-10-10 23:02:35,211 [Worker-7: Loading available Gradle versions] INFO o.e.b.c.i.u.g.PublishedGradleVersions - Gradle version information cache is out-of-date. Trying to update.
|
||||
Loaded 100 of 127 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user