This commit is contained in:
Celeste Madeline Samuel Nokto Dur Grossiord committed 2026-10-11 15:47:17 +02:00
1 parent 97649d73eb
commit e77afd4d29
127 files changed
+7320 -1886

No files matched your search

@@ -0,0 +1,150 @@
package org.sillyjirafe.db;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
public class Crawl_StateDB
{
String URL = new String();
String insertSql = "INSERT OR IGNORE INTO crawlstates (bind_name, last_page, rnd, error) VALUES (?, ?, ?, ?)";
Connection connection;
public Crawl_StateDB(String BDstr) throws SQLException
{
URL = BDstr;
connection = DriverManager.getConnection(URL);
}
public void LinkToDB() throws SQLException
{
try (Statement stmt = connection.createStatement())
{
stmt.execute("""
CREATE TABLE IF NOT EXISTS crawlstates (
bind_name TEXT PRIMARY KEY,
last_page INTEGER NOT NULL DEFAULT 1,
rnd TEXT NOT NULL DEFAULT 'a',
error INTEGER NOT NULL DEFAULT 0,
)
""");
}
}
public void AddEllement(String bind, int lastpage) throws SQLException
{
try (PreparedStatement ps = connection.prepareStatement(insertSql))
{
ps.setString(1, bind);
ps.setInt(2, lastpage);
ps.executeUpdate();
System.out.printf("%s is now page %d \n", bind, lastpage);
}
}
public int getPage(String bindName) throws SQLException {
String sql = "SELECT last_page FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getInt("last_page");
} else {
return 1;
}
}
}
public String getRND(String bindName) throws SQLException {
String sql = "SELECT rnd FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getString("rnd");
} else {
return "a";
}
}
}
public int getErrors(String bindName) throws SQLException {
String sql = "SELECT error FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getInt("error");
} else {
return 0;
}
}
}
public void updatePage(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET last_page = last_page + 1
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void newError(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET error = error + 1
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void resetError(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET error = 0
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void resetPage(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET last_page = 1
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void updateletter(String bindName) throws SQLException
{
if (getErrors(bindName) >= 10)
{
resetError(bindName);
resetPage(bindName);
}
}
public void Close() throws SQLException
{
if (connection != null)
{
connection.close();
}
}
}
@@ -0,0 +1,176 @@
package org.sillyjirafe.db;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
public class Crawl_StateDB
{
String URL = new String();
String insertSql = "INSERT OR IGNORE INTO crawlstates (bind_name, last_page, rnd, error) VALUES (?, ?, ?, ?)";
Connection connection;
public Crawl_StateDB(String BDstr) throws SQLException
{
URL = BDstr;
connection = DriverManager.getConnection(URL);
}
public void LinkToDB() throws SQLException
{
try (Statement stmt = connection.createStatement())
{
stmt.execute("""
CREATE TABLE IF NOT EXISTS crawlstates (
bind_name TEXT PRIMARY KEY,
last_page INTEGER NOT NULL DEFAULT 1,
rnd TEXT NOT NULL DEFAULT 'a',
error INTEGER NOT NULL DEFAULT 0,
)
""");
}
}
public void AddEllement(String bind, int lastpage) throws SQLException
{
try (PreparedStatement ps = connection.prepareStatement(insertSql))
{
ps.setString(1, bind);
ps.setInt(2, lastpage);
ps.executeUpdate();
System.out.printf("%s is now page %d \n", bind, lastpage);
}
}
public int getPage(String bindName) throws SQLException {
String sql = "SELECT last_page FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getInt("last_page");
} else {
return 1;
}
}
}
public String getRND(String bindName) throws SQLException {
String sql = "SELECT rnd FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getString("rnd");
} else {
return "a";
}
}
}
public int getErrors(String bindName) throws SQLException {
String sql = "SELECT error FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getInt("error");
} else {
return 0;
}
}
}
public void updatePage(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET last_page = last_page + 1
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void setRnd(String bindName, String c) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET last_page = (rnd)
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void newError(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET error = error + 1
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void resetError(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET error = 0
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void resetPage(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET last_page = 1
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public static String nextLetter(String current) {
char[] lettres = current.toCharArray();
int i = lettres.length - 1;
while (i >= 0) {
if (lettres[i] < 'z') {
lettres[i]++;
return new String(lettres);
}
lettres[i] = 'a';
i--;
}
return "a" + new String(lettres);
}
public void updateletter(String bindName) throws SQLException
{
if (getErrors(bindName) >= 10)
{
setRnd(bindName, nextLetter(getRND(bindName)));
resetError(bindName);
resetPage(bindName);
}
}
public void Close() throws SQLException
{
if (connection != null)
{
connection.close();
}
}
}
@@ -0,0 +1,44 @@
package org.sillyjirafe.uoapi.binds;
import java.util.ArrayList;
import java.util.List;
import org.json.JSONArray;
import org.json.JSONObject;
import org.sillyjirafe.uoapi.Api;
import org.sillyjirafe.uoapi.element;
public class PixabayImagesApi implements Api
{
public String getName()
{
return "PixabayImagesApi";
}
public String getURL(int page, String apikey, String rnd) {
return "https://pixabay.com/api/?per_page=200&page=" + page + "&key=" + apikey;
}
public List<element> Parse(String jsonRaw) {
List<element> resultats = new ArrayList<>();
try {
JSONObject json = new JSONObject(jsonRaw);
JSONArray items = json.getJSONArray("hits");
for (int i = 0; i < items.length(); i++) {
JSONObject repo = items.getJSONObject(i);
String titre = repo.getString("name");
String url = repo.getString("pageURL");
String licence = "cc0";
resultats.add(new element(titre, url, licence, "pixabay", "image"));
}
} catch (org.json.JSONException e) {
System.out.println("error but continue : " + e.getMessage());
}
return resultats;
}
}
@@ -0,0 +1,151 @@
package org.sillyjirafe.db;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
public class Crawl_StateDB
{
String URL = new String();
String insertSql = "INSERT OR IGNORE INTO crawlstates (bind_name, last_page, rnd, error) VALUES (?, ?, ?, ?)";
Connection connection;
public Crawl_StateDB(String BDstr) throws SQLException
{
URL = BDstr;
connection = DriverManager.getConnection(URL);
}
public void LinkToDB() throws SQLException
{
try (Statement stmt = connection.createStatement())
{
stmt.execute("""
CREATE TABLE IF NOT EXISTS crawlstates (
bind_name TEXT PRIMARY KEY,
last_page INTEGER NOT NULL DEFAULT 1,
rnd TEXT NOT NULL DEFAULT 'a',
error INTEGER NOT NULL DEFAULT 0,
)
""");
}
}
public void AddEllement(String bind, int lastpage) throws SQLException
{
try (PreparedStatement ps = connection.prepareStatement(insertSql))
{
ps.setString(1, bind);
ps.setInt(2, lastpage);
ps.setString(3, rnd);
ps.executeUpdate();
System.out.printf("%s is now page %d \n", bind, lastpage);
}
}
public int getPage(String bindName) throws SQLException {
String sql = "SELECT last_page FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getInt("last_page");
} else {
return 1;
}
}
}
public String getRND(String bindName) throws SQLException {
String sql = "SELECT rnd FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getString("rnd");
} else {
return "a";
}
}
}
public int getErrors(String bindName) throws SQLException {
String sql = "SELECT error FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getInt("error");
} else {
return 0;
}
}
}
public void updatePage(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET last_page = last_page + 1
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void newError(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET error = error + 1
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void resetError(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET error = 0
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void resetPage(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET last_page = 1
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void updateletter(String bindName) throws SQLException
{
if (getErrors(bindName) >= 10)
{
resetError(bindName);
resetPage(bindName);
}
}
public void Close() throws SQLException
{
if (connection != null)
{
connection.close();
}
}
}
@@ -0,0 +1,44 @@
package org.sillyjirafe.uoapi.binds;
import java.util.ArrayList;
import java.util.List;
import org.json.JSONArray;
import org.json.JSONObject;
import org.sillyjirafe.uoapi.Api;
import org.sillyjirafe.uoapi.element;
public class GitHubApacheApi implements Api
{
public String getName()
{
return "GitHubApacheApi";
}
public String getURL(int page, String apikey, String rnd) {
return "https://api.github.com/search/repositories?q="+ rnd +"&license:apache-2.0&page=" + page;
}
public List<element> Parse(String jsonRaw) {
List<element> resultats = new ArrayList<>();
try {
JSONObject json = new JSONObject(jsonRaw);
JSONArray items = json.getJSONArray("items");
for (int i = 0; i < items.length(); i++) {
JSONObject repo = items.getJSONObject(i);
String titre = repo.getString("full_name");
String url = repo.getString("html_url");
String licence = repo.getJSONObject("license").getString("spdx_id");
resultats.add(new element(titre, url, licence, "github", "code"));
}
} catch (org.json.JSONException e) {
System.out.println("error but continue : " + e.getMessage());
}
return resultats;
}
}
@@ -0,0 +1,46 @@
package org.sillyjirafe.uoapi;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.sillyjirafe.uoapi.binds.*;
public class Out {
List<Api> Apis = new ArrayList<>();
public List<String> PreOut()
{
Apis.add(new GitHubMITApi());
Apis.add(new GitHubLGPLApi());
Apis.add(new GitHubApacheApi());
Apis.add(new OpenVerseImagesApi());
Apis.add(new OpenVerseAudiosApi());
Apis.add(new PixabayImagesApi());
List<String> bindnames = new ArrayList<>();
for (Api api : Apis)
{
bindnames.add(api.getName());
}
return bindnames;
}
public List<element> out(Map<String, Integer> bindpages, Map<String, String> ApiKeys) throws Exception
{
List<element> Elements = new ArrayList<>();
for (int i = 0; i < Apis.size(); i++)
{
List<element> currentapi = Caller.CallApi(Apis.get(i), bindpages.get(Apis.get(i).getName()), ApiKeys.get(Apis.get(i).getName()));
for (int j = 0; j < currentapi.size(); j++)
{
Elements.add(currentapi.get(j));
}
}
return Elements;
}
}
@@ -0,0 +1,43 @@
package org.sillyjirafe.uoapi.binds;
import java.util.ArrayList;
import java.util.List;
import org.json.JSONArray;
import org.json.JSONObject;
import org.sillyjirafe.uoapi.Api;
import org.sillyjirafe.uoapi.element;
public class OpenVerseImagesApi implements Api
{
public String getName()
{
return "OpenVerseImagesApi";
}
public String getURL(int page, String apikey, String rnd) {
return "https://api.openverse.org/v1/images/?q=\n" + rnd+"\nformat=json&page=" + page;
}
public List<element> Parse(String jsonRaw) {
List<element> resultats = new ArrayList<>();
try {
JSONObject json = new JSONObject(jsonRaw);
JSONArray items = json.getJSONArray("results");
for (int i = 0; i < items.length(); i++) {
JSONObject image = items.getJSONObject(i);
String titre = image.getString("title");
String url = image.getString("url");
String licence = image.getString("license");
resultats.add(new element(titre, url, licence, "openverse", "image"));
}
} catch (org.json.JSONException e) {
System.out.println("error but continue : " + e.getMessage());
}
return resultats;
}
}
@@ -0,0 +1,41 @@
package org.sillyjirafe.uoapi.binds;
import java.util.ArrayList;
import java.util.List;
import org.json.JSONArray;
import org.json.JSONObject;
import org.sillyjirafe.uoapi.Api;
import org.sillyjirafe.uoapi.element;
public class OpenVerseImagesApi implements Api
{
public String getName()
{
return "OpenVerseImagesApi";
}
public String getURL(int page, String apikey, String rnd) {
return "https://api.openverse.org/v1/images/?q=\"" + rnd+"\"format=json&page=" + page;
}
public List<element> Parse(String jsonRaw) {
List<element> resultats = new ArrayList<>();
JSONObject json = new JSONObject(jsonRaw);
JSONArray items = json.getJSONArray("results");
for (int i = 0; i < items.length(); i++) {
JSONObject image = items.getJSONObject(i);
String titre = image.getString("title");
String url = image.getString("url");
String licence = image.getString("license");
resultats.add(new element(titre, url, licence, "openverse", "image"));
}
} catch (org.json.JSONException e) {
System.out.println("error but continue : " + e.getMessage());
}
return resultats;
}
}
@@ -0,0 +1,43 @@
package org.sillyjirafe.uoapi.binds;
import java.util.ArrayList;
import java.util.List;
import org.json.JSONArray;
import org.json.JSONObject;
import org.sillyjirafe.uoapi.Api;
import org.sillyjirafe.uoapi.element;
public class OpenVerseAudiosApi implements Api
{
public String getName()
{
return "OpenVerseAudiosApi";
}
public String getURL(int page, String apikey, String rnd) {
return "https://api.openverse.org/v1/audio/?q=\""+ rnd +"\"format=json&page=" + page;
}
public List<element> Parse(String jsonRaw) {
List<element> resultats = new ArrayList<>();
try {
JSONObject json = new JSONObject(jsonRaw);
JSONArray items = json.getJSONArray("results");
for (int i = 0; i < items.length(); i++) {
JSONObject audio = items.getJSONObject(i);
String titre = audio.getString("title");
String url = audio.getString("url");
String licence = audio.getString("license");
resultats.add(new element(titre, url, licence, "openverse", "audio"));
}
} catch (org.json.JSONException e) {
System.out.println("error but continue : " + e.getMessage());
}
return resultats;
}
}
@@ -0,0 +1,164 @@
package org.sillyjirafe.db;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
public class Crawl_StateDB
{
String URL = new String();
String insertSql = "INSERT OR IGNORE INTO crawlstates (bind_name, last_page, rnd, error) VALUES (?, ?, ?, ?)";
Connection connection;
public Crawl_StateDB(String BDstr) throws SQLException
{
URL = BDstr;
connection = DriverManager.getConnection(URL);
}
public void LinkToDB() throws SQLException
{
try (Statement stmt = connection.createStatement())
{
stmt.execute("""
CREATE TABLE IF NOT EXISTS crawlstates (
bind_name TEXT PRIMARY KEY,
last_page INTEGER NOT NULL DEFAULT 1,
rnd TEXT NOT NULL DEFAULT 'a',
error INTEGER NOT NULL DEFAULT 0,
)
""");
}
}
public void AddEllement(String bind, int lastpage) throws SQLException
{
try (PreparedStatement ps = connection.prepareStatement(insertSql))
{
ps.setString(1, bind);
ps.setInt(2, lastpage);
ps.executeUpdate();
System.out.printf("%s is now page %d \n", bind, lastpage);
}
}
public int getPage(String bindName) throws SQLException {
String sql = "SELECT last_page FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getInt("last_page");
} else {
return 1;
}
}
}
public String getRND(String bindName) throws SQLException {
String sql = "SELECT rnd FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getString("rnd");
} else {
return "a";
}
}
}
public int getErrors(String bindName) throws SQLException {
String sql = "SELECT error FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getInt("error");
} else {
return 0;
}
}
}
public void updatePage(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET last_page = last_page + 1
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void newError(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET error = error + 1
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void resetError(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET error = 0
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void resetPage(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET last_page = 1
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public static String nextletter(String actuelle) {
char[] lettres = actuelle.toCharArray();
int i = lettres.length - 1;
while (i >= 0) {
if (lettres[i] < 'z') {
lettres[i]++;
return new String(lettres);
}
lettres[i] = 'a';
i--;
}
return "a" + new String(lettres);
}
public void updateletter(String bindName) throws SQLException
{
if (getErrors(bindName) >= 10)
{
resetError(bindName);
resetPage(bindName);
}
}
public void Close() throws SQLException
{
if (connection != null)
{
connection.close();
}
}
}
@@ -0,0 +1,150 @@
package org.sillyjirafe.db;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
public class Crawl_StateDB
{
String URL = new String();
String insertSql = "INSERT OR IGNORE INTO crawlstates (bind_name, last_page, rnd, error) VALUES (?, ?, ?, ?)";
Connection connection;
public Crawl_StateDB(String BDstr) throws SQLException
{
URL = BDstr;
connection = DriverManager.getConnection(URL);
}
public void LinkToDB() throws SQLException
{
try (Statement stmt = connection.createStatement())
{
stmt.execute("""
CREATE TABLE IF NOT EXISTS crawlstates (
bind_name TEXT PRIMARY KEY,
last_page INTEGER NOT NULL DEFAULT 1,
rnd TEXT NOT NULL DEFAULT 'a',
error INTEGER NOT NULL DEFAULT 0,
)
""");
}
}
public void AddEllement(String bind, int lastpage) throws SQLException
{
try (PreparedStatement ps = connection.prepareStatement(insertSql))
{
ps.setString(1, bind);
ps.setInt(2, lastpage);
ps.executeUpdate();
System.out.printf("%s is now page %d \n", bind, lastpage);
}
}
public int getPage(String bindName) throws SQLException {
String sql = "SELECT last_page FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getInt("last_page");
} else {
return 1;
}
}
}
public String getRND(String bindName) throws SQLException {
String sql = "SELECT rnd FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getString("rnd");
} else {
return "a";
}
}
}
public int getErrors(String bindName) throws SQLException {
String sql = "SELECT error FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getInt("error");
} else {
return 0;
}
}
}
public void updatePage(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET last_page = last_page + 1
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void newError(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET error = error + 1
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void resetError(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET error = 0
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void resetPage(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET last_page = 0
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void updateletter(String bindName) throws SQLException
{
if (getErrors(bindName) >= 10)
{
resetError(bindName);
resetPage(bindName);
}
}
public void Close() throws SQLException
{
if (connection != null)
{
connection.close();
}
}
}
@@ -0,0 +1,43 @@
package org.sillyjirafe.uoapi.binds;
import java.util.ArrayList;
import java.util.List;
import org.json.JSONArray;
import org.json.JSONObject;
import org.sillyjirafe.uoapi.Api;
import org.sillyjirafe.uoapi.element;
public class OpenVerseImagesApi implements Api
{
public String getName()
{
return "OpenVerseImagesApi";
}
public String getURL(int page, String apikey) {
return "https://api.openverse.org/v1/images/?format=json&page=" + page;
}
public List<element> Parse(String jsonRaw) {
List<element> resultats = new ArrayList<>();
try {
JSONObject json = new JSONObject(jsonRaw);
JSONArray items = json.getJSONArray("results");
for (int i = 0; i < items.length(); i++) {
JSONObject image = items.getJSONObject(i);
String titre = image.getString("title");
String url = image.getString("url");
String licence = image.getString("license");
resultats.add(new element(titre, url, licence, "openverse", "image"));
}
} catch (org.json.JSONException e) {
System.out.println("error but continue : " + e.getMessage());
}
return resultats;
}
}
@@ -0,0 +1,138 @@
package org.sillyjirafe.db;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
public class Crawl_StateDB
{
String URL = new String();
String insertSql = "INSERT OR IGNORE INTO crawlstates (bind_name, last_page, rnd, error) VALUES (?, ?, ?, ?)";
Connection connection;
public Crawl_StateDB(String BDstr) throws SQLException
{
URL = BDstr;
connection = DriverManager.getConnection(URL);
}
public void LinkToDB() throws SQLException
{
try (Statement stmt = connection.createStatement())
{
stmt.execute("""
CREATE TABLE IF NOT EXISTS crawlstates (
bind_name TEXT PRIMARY KEY,
last_page INTEGER NOT NULL DEFAULT 1,
rnd TEXT NOT NULL DEFAULT 'a'
error INTEGER NOT NULL DEFAULT 0,
)
""");
}
}
public void AddEllement(String bind, int lastpage) throws SQLException
{
try (PreparedStatement ps = connection.prepareStatement(insertSql))
{
ps.setString(1, bind);
ps.setInt(2, lastpage);
ps.executeUpdate();
System.out.printf("%s is now page %d \n", bind, lastpage);
}
}
public int getPage(String bindName) throws SQLException {
String sql = "SELECT last_page FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getInt("last_page");
} else {
return 1;
}
}
}
public String getRND(String bindName) throws SQLException {
String sql = "SELECT rnd FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getString("rnd");
} else {
return "a";
}
}
}
public int getErrors(String bindName) throws SQLException {
String sql = "SELECT error FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getInt("error");
} else {
return 0;
}
}
}
public void updatePage(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, rnd) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET rnd = rnd + 1
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void newError(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET error = error + 1
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void resetError(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET error = 0
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void updateletter(String bindName) throws SQLException
{
if (getErrors(bindName) >= 10)
{
}
}
public void Close() throws SQLException
{
if (connection != null)
{
connection.close();
}
}
}
@@ -0,0 +1,150 @@
package org.sillyjirafe.db;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
public class Crawl_StateDB
{
String URL = new String();
String insertSql = "INSERT OR IGNORE INTO crawlstates (bind_name, last_page, rnd, error) VALUES (?, ?, ?, ?)";
Connection connection;
public Crawl_StateDB(String BDstr) throws SQLException
{
URL = BDstr;
connection = DriverManager.getConnection(URL);
}
public void LinkToDB() throws SQLException
{
try (Statement stmt = connection.createStatement())
{
stmt.execute("""
CREATE TABLE IF NOT EXISTS crawlstates (
bind_name TEXT PRIMARY KEY,
last_page INTEGER NOT NULL DEFAULT 1,
rnd TEXT NOT NULL DEFAULT 'a',
error INTEGER NOT NULL DEFAULT 0,
)
""");
}
}
public void AddEllement(String bind, int lastpage) throws SQLException
{
try (PreparedStatement ps = connection.prepareStatement(insertSql))
{
ps.setString(1, bind);
ps.setInt(2, lastpage);
ps.executeUpdate();
System.out.printf("%s is now page %d \n", bind, lastpage);
}
}
public int getPage(String bindName) throws SQLException {
String sql = "SELECT last_page FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getInt("last_page");
} else {
return 1;
}
}
}
public String getRND(String bindName) throws SQLException {
String sql = "SELECT rnd FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getString("rnd");
} else {
return "a";
}
}
}
public int getErrors(String bindName) throws SQLException {
String sql = "SELECT error FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getInt("error");
} else {
return 0;
}
}
}
public void updatePage(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET last_page = last_page + 1
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void newError(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET error = error + 1
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void resetError(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET error = 0
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void resetPage(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET last_page = 1
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void updateletter(String bindName) throws SQLException
{
if (getErrors(bindName) >= 10)
{
resetError(bindName);
resetPage(bindName);
}
}
public void Close() throws SQLException
{
if (connection != null)
{
connection.close();
}
}
}
@@ -0,0 +1,43 @@
package org.sillyjirafe.uoapi.binds;
import java.util.ArrayList;
import java.util.List;
import org.json.JSONArray;
import org.json.JSONObject;
import org.sillyjirafe.uoapi.Api;
import org.sillyjirafe.uoapi.element;
public class OpenVerseImagesApi implements Api
{
public String getName()
{
return "OpenVerseImagesApi";
}
public String getURL(int page, String apikey, String rnd) {
return "https://api.openverse.org/v1/images/?format=json&page=" + page;
}
public List<element> Parse(String jsonRaw) {
List<element> resultats = new ArrayList<>();
try {
JSONObject json = new JSONObject(jsonRaw);
JSONArray items = json.getJSONArray("results");
for (int i = 0; i < items.length(); i++) {
JSONObject image = items.getJSONObject(i);
String titre = image.getString("title");
String url = image.getString("url");
String licence = image.getString("license");
resultats.add(new element(titre, url, licence, "openverse", "image"));
}
} catch (org.json.JSONException e) {
System.out.println("error but continue : " + e.getMessage());
}
return resultats;
}
}
@@ -0,0 +1,55 @@
package org.sillyjirafe.linker;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.sillyjirafe.db.Crawl_StateDB;
import org.sillyjirafe.db.ElementsDB;
import org.sillyjirafe.json.ApiKeys;
import org.sillyjirafe.uoapi.*;
public class Main {
public static void main(String[] args) throws Exception
{
Out Out = new Out();
ElementsDB elementDB = new ElementsDB("jdbc:sqlite:elements.db");
Crawl_StateDB crawlstateDB = new Crawl_StateDB("jdbc:sqlite:crawlstates.db");
elementDB.LinkToDB();
crawlstateDB.LinkToDB();
Map<String, Boolean> succes = Out.getSucces();
List<String> bindNames = Out.PreOut();
Map<String, Integer> pages = new HashMap<>();
for (String name : bindNames) {
pages.put(name, crawlstateDB.getPage(name));
}
Map<String, String> rnds = new HashMap<>();
for (String nom : bindNames) {
rnds.put(nom, crawlstateDB.getRND(nom));
}
List<element> elements = Out.out(pages, ApiKeys.read(), rnds);
for (int i = 0; i < elements.size(); i++)
{
element el = elements.get(i);
elementDB.AddEllement(el.title(), el.url(), el.licence(), el.source(), el.type());
}
for (String name : bindNames) {
if (succes.getOrDefault(name, false)) {
crawlstateDB.updatePage(name);
} else {
crawlstateDB.addError(name);
}
}
elementDB.Close();
crawlstateDB.Close();
}
}
@@ -0,0 +1,49 @@
package org.sillyjirafe.linker;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.sillyjirafe.db.Crawl_StateDB;
import org.sillyjirafe.db.ElementsDB;
import org.sillyjirafe.json.ApiKeys;
import org.sillyjirafe.uoapi.*;
public class Main {
public static void main(String[] args) throws Exception
{
Out Out = new Out();
ElementsDB elementDB = new ElementsDB("jdbc:sqlite:elements.db");
Crawl_StateDB crawlstateDB = new Crawl_StateDB("jdbc:sqlite:crawlstates.db");
elementDB.LinkToDB();
crawlstateDB.LinkToDB();
List<String> bindNames = Out.PreOut();
Map<String, Integer> pages = new HashMap<>();
for (String nom : bindNames) {
pages.put(nom, crawlstateDB.getPage(nom));
}
Map<String, String> rnds = new HashMap<>();
for (String nom : bindNames) {
pages.put(nom, crawlstateDB.getPage(nom));
}
List<element> elements = Out.out(pages, ApiKeys.read());
for (int i = 0; i < elements.size(); i++)
{
element el = elements.get(i);
elementDB.AddEllement(el.title(), el.url(), el.licence(), el.source(), el.type());
}
for (String name : bindNames) {
crawlstateDB.updatePage(name);
}
elementDB.Close();
crawlstateDB.Close();
}
}
@@ -0,0 +1,98 @@
package org.sillyjirafe.db;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
public class Crawl_StateDB
{
String URL = new String();
String insertSql = "INSERT OR IGNORE INTO crawlstates (bind_name, last_page, rnd, error) VALUES (?, ?, ?, ?)";
Connection connection;
public Crawl_StateDB(String BDstr) throws SQLException
{
URL = BDstr;
connection = DriverManager.getConnection(URL);
}
public void LinkToDB() throws SQLException
{
try (Statement stmt = connection.createStatement())
{
stmt.execute("""
CREATE TABLE IF NOT EXISTS crawlstates (
bind_name TEXT PRIMARY KEY,
last_page INTEGER NOT NULL DEFAULT 1,
rnd TEXT NOT NULL DEFAULT 'a'
)
""");
}
}
public void AddEllement(String bind, int lastpage) throws SQLException
{
try (PreparedStatement ps = connection.prepareStatement(insertSql))
{
ps.setString(1, bind);
ps.setInt(2, lastpage);
ps.executeUpdate();
System.out.printf("%s is now page %d \n", bind, lastpage);
}
}
public int getPage(String bindName) throws SQLException {
String sql = "SELECT last_page FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getInt("last_page");
} else {
return 1;
}
}
}
public String getRND(String bindName) throws SQLException {
String sql = "SELECT rnd FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getString("rnd");
} else {
return "a";
}
}
}
public void updatePage(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET last_page = last_page + 1
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void updateletter(String bindName)
{
}
public void Close() throws SQLException
{
if (connection != null)
{
connection.close();
}
}
}
@@ -0,0 +1,175 @@
package org.sillyjirafe.db;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
public class Crawl_StateDB
{
String URL = new String();
String insertSql = "INSERT OR IGNORE INTO crawlstates (bind_name, last_page, rnd, error) VALUES (?, ?, ?, ?)";
Connection connection;
public Crawl_StateDB(String BDstr) throws SQLException
{
URL = BDstr;
connection = DriverManager.getConnection(URL);
}
public void LinkToDB() throws SQLException
{
try (Statement stmt = connection.createStatement())
{
stmt.execute("""
CREATE TABLE IF NOT EXISTS crawlstates (
bind_name TEXT PRIMARY KEY,
last_page INTEGER NOT NULL DEFAULT 1,
rnd TEXT NOT NULL DEFAULT 'a',
error INTEGER NOT NULL DEFAULT 0,
)
""");
}
}
public void AddEllement(String bind, int lastpage) throws SQLException
{
try (PreparedStatement ps = connection.prepareStatement(insertSql))
{
ps.setString(1, bind);
ps.setInt(2, lastpage);
ps.executeUpdate();
System.out.printf("%s is now page %d \n", bind, lastpage);
}
}
public int getPage(String bindName) throws SQLException {
String sql = "SELECT last_page FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getInt("last_page");
} else {
return 1;
}
}
}
public String getRND(String bindName) throws SQLException {
String sql = "SELECT rnd FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getString("rnd");
} else {
return "a";
}
}
}
public int getErrors(String bindName) throws SQLException {
String sql = "SELECT error FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getInt("error");
} else {
return 0;
}
}
}
public void updatePage(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET last_page = last_page + 1
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void setRnd(String bindName, String c) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET last_page = last_page + 1
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void newError(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET error = error + 1
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void resetError(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET error = 0
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void resetPage(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET last_page = 1
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public static String nextletter(String current) {
char[] lettres = current.toCharArray();
int i = lettres.length - 1;
while (i >= 0) {
if (lettres[i] < 'z') {
lettres[i]++;
return new String(lettres);
}
lettres[i] = 'a';
i--;
}
return "a" + new String(lettres);
}
public void updateletter(String bindName) throws SQLException
{
if (getErrors(bindName) >= 10)
{
resetError(bindName);
resetPage(bindName);
}
}
public void Close() throws SQLException
{
if (connection != null)
{
connection.close();
}
}
}
@@ -0,0 +1,175 @@
package org.sillyjirafe.db;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
public class Crawl_StateDB
{
String URL = new String();
String insertSql = "INSERT OR IGNORE INTO crawlstates (bind_name, last_page, rnd, error) VALUES (?, ?, ?, ?)";
Connection connection;
public Crawl_StateDB(String BDstr) throws SQLException
{
URL = BDstr;
connection = DriverManager.getConnection(URL);
}
public void LinkToDB() throws SQLException
{
try (Statement stmt = connection.createStatement())
{
stmt.execute("""
CREATE TABLE IF NOT EXISTS crawlstates (
bind_name TEXT PRIMARY KEY,
last_page INTEGER NOT NULL DEFAULT 1,
rnd TEXT NOT NULL DEFAULT 'a',
error INTEGER NOT NULL DEFAULT 0,
)
""");
}
}
public void AddEllement(String bind, int lastpage) throws SQLException
{
try (PreparedStatement ps = connection.prepareStatement(insertSql))
{
ps.setString(1, bind);
ps.setInt(2, lastpage);
ps.executeUpdate();
System.out.printf("%s is now page %d \n", bind, lastpage);
}
}
public int getPage(String bindName) throws SQLException {
String sql = "SELECT last_page FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getInt("last_page");
} else {
return 1;
}
}
}
public String getRND(String bindName) throws SQLException {
String sql = "SELECT rnd FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getString("rnd");
} else {
return "a";
}
}
}
public int getErrors(String bindName) throws SQLException {
String sql = "SELECT error FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getInt("error");
} else {
return 0;
}
}
}
public void updatePage(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET last_page = last_page + 1
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void setRnd(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET last_page = last_page + 1
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void newError(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET error = error + 1
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void resetError(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET error = 0
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void resetPage(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET last_page = 1
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public static String nextletter(String current) {
char[] lettres = current.toCharArray();
int i = lettres.length - 1;
while (i >= 0) {
if (lettres[i] < 'z') {
lettres[i]++;
return new String(lettres);
}
lettres[i] = 'a';
i--;
}
return "a" + new String(lettres);
}
public void updateletter(String bindName) throws SQLException
{
if (getErrors(bindName) >= 10)
{
resetError(bindName);
resetPage(bindName);
}
}
public void Close() throws SQLException
{
if (connection != null)
{
connection.close();
}
}
}
@@ -0,0 +1,46 @@
package org.sillyjirafe.uoapi;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.sillyjirafe.uoapi.binds.*;
public class Out {
List<Api> Apis = new ArrayList<>();
public List<String> PreOut()
{
Apis.add(new GitHubMITApi());
Apis.add(new GitHubLGPLApi());
Apis.add(new GitHubApacheApi());
Apis.add(new OpenVerseImagesApi());
Apis.add(new OpenVerseAudiosApi());
Apis.add(new PixabayImagesApi());
List<String> bindnames = new ArrayList<>();
for (Api api : Apis)
{
bindnames.add(api.getName());
}
return bindnames;
}
public List<element> out(Map<String, Integer> bindpages, Map<String, String> ApiKeys, Map<String, String> Rnds) throws Exception
{
List<element> Elements = new ArrayList<>();
for (int i = 0; i < Apis.size(); i++)
{
List<element> currentapi = Caller.CallApi(Apis.get(i), bindpages.get(Apis.get(i).getName()), ApiKeys.get(Apis.get(i).getName()), );
for (int j = 0; j < currentapi.size(); j++)
{
Elements.add(currentapi.get(j));
}
}
return Elements;
}
}
@@ -0,0 +1,164 @@
package org.sillyjirafe.db;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
public class Crawl_StateDB
{
String URL = new String();
String insertSql = "INSERT OR IGNORE INTO crawlstates (bind_name, last_page, rnd, error) VALUES (?, ?, ?, ?)";
Connection connection;
public Crawl_StateDB(String BDstr) throws SQLException
{
URL = BDstr;
connection = DriverManager.getConnection(URL);
}
public void LinkToDB() throws SQLException
{
try (Statement stmt = connection.createStatement())
{
stmt.execute("""
CREATE TABLE IF NOT EXISTS crawlstates (
bind_name TEXT PRIMARY KEY,
last_page INTEGER NOT NULL DEFAULT 1,
rnd TEXT NOT NULL DEFAULT 'a',
error INTEGER NOT NULL DEFAULT 0,
)
""");
}
}
public void AddEllement(String bind, int lastpage) throws SQLException
{
try (PreparedStatement ps = connection.prepareStatement(insertSql))
{
ps.setString(1, bind);
ps.setInt(2, lastpage);
ps.executeUpdate();
System.out.printf("%s is now page %d \n", bind, lastpage);
}
}
public int getPage(String bindName) throws SQLException {
String sql = "SELECT last_page FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getInt("last_page");
} else {
return 1;
}
}
}
public String getRND(String bindName) throws SQLException {
String sql = "SELECT rnd FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getString("rnd");
} else {
return "a";
}
}
}
public int getErrors(String bindName) throws SQLException {
String sql = "SELECT error FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getInt("error");
} else {
return 0;
}
}
}
public void updatePage(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET last_page = last_page + 1
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void newError(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET error = error + 1
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void resetError(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET error = 0
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void resetPage(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET last_page = 1
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public static String lettreSuivante(String actuelle) {
char[] lettres = actuelle.toCharArray();
int i = lettres.length - 1;
while (i >= 0) {
if (lettres[i] < 'z') {
lettres[i]++;
return new String(lettres);
}
lettres[i] = 'a';
i--;
}
return "a" + new String(lettres);
}
public void updateletter(String bindName) throws SQLException
{
if (getErrors(bindName) >= 10)
{
resetError(bindName);
resetPage(bindName);
}
}
public void Close() throws SQLException
{
if (connection != null)
{
connection.close();
}
}
}
@@ -0,0 +1,99 @@
package org.sillyjirafe.db;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
public class Crawl_StateDB
{
String URL = new String();
String insertSql = "INSERT OR IGNORE INTO crawlstates (bind_name, last_page, rnd, error) VALUES (?, ?, ?, ?)";
Connection connection;
public Crawl_StateDB(String BDstr) throws SQLException
{
URL = BDstr;
connection = DriverManager.getConnection(URL);
}
public void LinkToDB() throws SQLException
{
try (Statement stmt = connection.createStatement())
{
stmt.execute("""
CREATE TABLE IF NOT EXISTS crawlstates (
bind_name TEXT PRIMARY KEY,
last_page INTEGER NOT NULL DEFAULT 1,
rnd TEXT NOT NULL DEFAULT 'a'
error INTEGER NOT NULL DEFAULT 0,
)
""");
}
}
public void AddEllement(String bind, int lastpage) throws SQLException
{
try (PreparedStatement ps = connection.prepareStatement(insertSql))
{
ps.setString(1, bind);
ps.setInt(2, lastpage);
ps.executeUpdate();
System.out.printf("%s is now page %d \n", bind, lastpage);
}
}
public int getPage(String bindName) throws SQLException {
String sql = "SELECT last_page FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getInt("last_page");
} else {
return 1;
}
}
}
public String getRND(String bindName) throws SQLException {
String sql = "SELECT rnd FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getString("rnd");
} else {
return "a";
}
}
}
public void updatePage(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET last_page = last_page + 1
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void updateletter(String bindName)
{
}
public void Close() throws SQLException
{
if (connection != null)
{
connection.close();
}
}
}
@@ -0,0 +1,177 @@
package org.sillyjirafe.db;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
public class Crawl_StateDB
{
String URL = new String();
String insertSql = "INSERT OR IGNORE INTO crawlstates (bind_name, last_page, rnd, error) VALUES (?, ?, ?, ?)";
Connection connection;
public Crawl_StateDB(String BDstr) throws SQLException
{
URL = BDstr;
connection = DriverManager.getConnection(URL);
}
public void LinkToDB() throws SQLException
{
try (Statement stmt = connection.createStatement())
{
stmt.execute("""
CREATE TABLE IF NOT EXISTS crawlstates (
bind_name TEXT PRIMARY KEY,
last_page INTEGER NOT NULL DEFAULT 1,
rnd TEXT NOT NULL DEFAULT 'a',
error INTEGER NOT NULL DEFAULT 0,
)
""");
}
}
public void AddEllement(String bind, int lastpage) throws SQLException
{
try (PreparedStatement ps = connection.prepareStatement(insertSql))
{
ps.setString(1, bind);
ps.setInt(2, lastpage);
ps.executeUpdate();
System.out.printf("%s is now page %d \n", bind, lastpage);
}
}
public int getPage(String bindName) throws SQLException {
String sql = "SELECT last_page FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getInt("last_page");
} else {
return 1;
}
}
}
public String getRND(String bindName) throws SQLException {
String sql = "SELECT rnd FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getString("rnd");
} else {
return "a";
}
}
}
public int getErrors(String bindName) throws SQLException {
String sql = "SELECT error FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getInt("error");
} else {
return 0;
}
}
}
public void updatePage(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET last_page = last_page + 1
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void setRnd(String bindName, String c) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, rnd) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET rnd = (rnd)
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.setString(2, c);
ps.executeUpdate();
}
}
public void newError(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET error = error + 1
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void resetError(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET error = 0
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void resetPage(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET last_page = 1
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public static String nextLetter(String current) {
char[] lettres = current.toCharArray();
int i = lettres.length - 1;
while (i >= 0) {
if (lettres[i] < 'z') {
lettres[i]++;
return new String(lettres);
}
lettres[i] = 'a';
i--;
}
return "a" + new String(lettres);
}
public void updateletter(String bindName) throws SQLException
{
if (getErrors(bindName) >= 10)
{
setRnd(bindName, nextLetter(getRND(bindName)));
resetError(bindName);
resetPage(bindName);
}
}
public void Close() throws SQLException
{
if (connection != null)
{
connection.close();
}
}
}
@@ -0,0 +1,50 @@
package org.sillyjirafe.uoapi;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.sillyjirafe.uoapi.binds.*;
public class Out {
List<Api> Apis = new ArrayList<>();
Map<String, Boolean> Succes = new HashMap<>();
public List<String> PreOut()
{
Apis.add(new GitHubMITApi());
Apis.add(new GitHubLGPLApi());
Apis.add(new GitHubApacheApi());
Apis.add(new OpenVerseImagesApi());
Apis.add(new OpenVerseAudiosApi());
Apis.add(new PixabayImagesApi());
List<String> bindnames = new ArrayList<>();
for (Api api : Apis)
{
bindnames.add(api.getName());
}
return bindnames;
}
public List<element> out(Map<String, Integer> bindpages, Map<String, String> ApiKeys, Map<String, String> Rnds) throws Exception
{
List<element> Elements = new ArrayList<>();
for (Api api : Apis) {
try {
List<element> currentapi = Caller.CallApi(api, bindpages.get(api.getName()));
Elements.addAll(currentapi);
Succes.put(api.getName(), true);
} catch (Exception e) {
System.out.println("Erreur pour " + api.getName() + " : " + e.getMessage());
Succes.put(api.getName(), false);
}
}
return Elements;
}
}
@@ -0,0 +1,45 @@
package org.sillyjirafe.uoapi.binds;
import java.util.ArrayList;
import java.util.List;
import org.json.JSONArray;
import org.json.JSONObject;
import org.sillyjirafe.uoapi.Api;
import org.sillyjirafe.uoapi.element;
public class GitHubLGPLApi implements Api
{
public String getName()
{
return "GitHubLGPLApi";
}
public String getURL(int page, String apikey) {
return "https://api.github.com/search/repositories?q=license:lgpl-3.0&page=" + page;
}
public List<element> Parse(String jsonRaw) {
List<element> resultats = new ArrayList<>();
try {
JSONObject json = new JSONObject(jsonRaw);
JSONArray items = json.getJSONArray("items");
for (int i = 0; i < items.length(); i++) {
JSONObject repo = items.getJSONObject(i);
String titre = repo.getString("full_name");
String url = repo.getString("html_url");
String licence = repo.getJSONObject("license").getString("spdx_id");
resultats.add(new element(titre, url, licence, "github", "code"));
}
} catch (org.json.JSONException e) {
System.out.println("error but continue : " + e.getMessage());
}
return resultats;
}
}
@@ -0,0 +1,50 @@
package org.sillyjirafe.uoapi;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.sillyjirafe.uoapi.binds.*;
public class Out {
List<Api> Apis = new ArrayList<>();
Map<String, Boolean> Succes = new HashMap<>();
public List<String> PreOut()
{
Apis.add(new GitHubMITApi());
Apis.add(new GitHubLGPLApi());
Apis.add(new GitHubApacheApi());
Apis.add(new OpenVerseImagesApi());
Apis.add(new OpenVerseAudiosApi());
Apis.add(new PixabayImagesApi());
List<String> bindnames = new ArrayList<>();
for (Api api : Apis)
{
bindnames.add(api.getName());
}
return bindnames;
}
public List<element> out(Map<String, Integer> bindpages, Map<String, String> ApiKeys, Map<String, String> Rnds) throws Exception
{
List<element> Elements = new ArrayList<>();
for (Api api : Apis) {
try {
List<element> currentapi = Caller.CallApi(api, bindpages.get(api.getName()), ApiKeys.get(api.getName()), Rnds.get(api.getName()));
Elements.addAll(currentapi);
Succes.put(api.getName(), true);
} catch (Exception e) {
System.out.println("Erreur pour " + api.getName() + " : " + e.getMessage());
Succes.put(api.getName(), false);
}
}
return Elements;
}
}
@@ -0,0 +1,44 @@
package org.sillyjirafe.linker;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.sillyjirafe.db.Crawl_StateDB;
import org.sillyjirafe.db.ElementsDB;
import org.sillyjirafe.json.ApiKeys;
import org.sillyjirafe.uoapi.*;
public class Main {
public static void main(String[] args) throws Exception
{
Out Out = new Out();
ElementsDB elementDB = new ElementsDB("jdbc:sqlite:elements.db");
Crawl_StateDB crawlstateDB = new Crawl_StateDB("jdbc:sqlite:crawlstates.db");
elementDB.LinkToDB();
crawlstateDB.LinkToDB();
List<String> bindNames = Out.PreOut();
Map<String, Integer> pages = new HashMap<>();
for (String nom : bindNames) {
pages.put(nom, crawlstateDB.getPage(nom));
}
List<element> elements = Out.out(pages, ApiKeys.read());
for (int i = 0; i < elements.size(); i++)
{
element el = elements.get(i);
elementDB.AddEllement(el.title(), el.url(), el.licence(), el.source(), el.type());
}
for (String name : bindNames) {
crawlstateDB.updatePage(name);
}
elementDB.Close();
crawlstateDB.Close();
}
}
@@ -0,0 +1,45 @@
package org.sillyjirafe.uoapi.binds;
import java.util.ArrayList;
import java.util.List;
import org.json.JSONArray;
import org.json.JSONObject;
import org.sillyjirafe.uoapi.Api;
import org.sillyjirafe.uoapi.element;
public class GitHubLGPLApi implements Api
{
public String getName()
{
return "GitHubLGPLApi";
}
public String getURL(int page, String apikey, String rnd) {
return "https://api.github.com/search/repositories?q=license:lgpl-3.0&page=" + page;
}
public List<element> Parse(String jsonRaw) {
List<element> resultats = new ArrayList<>();
try {
JSONObject json = new JSONObject(jsonRaw);
JSONArray items = json.getJSONArray("items");
for (int i = 0; i < items.length(); i++) {
JSONObject repo = items.getJSONObject(i);
String titre = repo.getString("full_name");
String url = repo.getString("html_url");
String licence = repo.getJSONObject("license").getString("spdx_id");
resultats.add(new element(titre, url, licence, "github", "code"));
}
} catch (org.json.JSONException e) {
System.out.println("error but continue : " + e.getMessage());
}
return resultats;
}
}
@@ -0,0 +1,177 @@
package org.sillyjirafe.db;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
public class Crawl_StateDB
{
String URL = new String();
String insertSql = "INSERT OR IGNORE INTO crawlstates (bind_name, last_page, rnd, error) VALUES (?, ?, ?, ?)";
Connection connection;
public Crawl_StateDB(String BDstr) throws SQLException
{
URL = BDstr;
connection = DriverManager.getConnection(URL);
}
public void LinkToDB() throws SQLException
{
try (Statement stmt = connection.createStatement())
{
stmt.execute("""
CREATE TABLE IF NOT EXISTS crawlstates (
bind_name TEXT PRIMARY KEY,
last_page INTEGER NOT NULL DEFAULT 1,
rnd TEXT NOT NULL DEFAULT 'a',
error INTEGER NOT NULL DEFAULT 0,
)
""");
}
}
public void AddEllement(String bind, int lastpage) throws SQLException
{
try (PreparedStatement ps = connection.prepareStatement(insertSql))
{
ps.setString(1, bind);
ps.setInt(2, lastpage);
ps.executeUpdate();
System.out.printf("%s is now page %d \n", bind, lastpage);
}
}
public int getPage(String bindName) throws SQLException {
String sql = "SELECT last_page FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getInt("last_page");
} else {
return 1;
}
}
}
public String getRND(String bindName) throws SQLException {
String sql = "SELECT rnd FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getString("rnd");
} else {
return "a";
}
}
}
public int getErrors(String bindName) throws SQLException {
String sql = "SELECT error FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getInt("error");
} else {
return 0;
}
}
}
public void updatePage(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET last_page = last_page + 1
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void setRnd(String bindName, String c) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, rnd) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET rnd = (rnd)
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void newError(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET error = error + 1
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void resetError(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET error = 0
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void resetPage(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET last_page = 1
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public static String nextLetter(String current) {
char[] lettres = current.toCharArray();
int i = lettres.length - 1;
while (i >= 0) {
if (lettres[i] < 'z') {
lettres[i]++;
return new String(lettres);
}
lettres[i] = 'a';
i--;
}
return "a" + new String(lettres);
}
public void updateletter(String bindName) throws SQLException
{
if (getErrors(bindName) >= 10)
{
setRnd(bindName, nextLetter(getRND(bindName)));
resetError(bindName);
resetPage(bindName);
}
}
public void Close() throws SQLException
{
if (connection != null)
{
connection.close();
}
}
}
@@ -0,0 +1,45 @@
package org.sillyjirafe.uoapi.binds;
import org.json.JSONArray;
import org.json.JSONObject;
import org.sillyjirafe.uoapi.Api;
import org.sillyjirafe.uoapi.element;
import java.util.ArrayList;
import java.util.List;
public class GitHubMITApi implements Api
{
public String getName()
{
return "GitHubMITApi";
}
public String getURL(int page, String apikey) {
return "https://api.github.com/search/repositories?q=license:mit&page=" + page;
}
public List<element> Parse(String jsonRaw) {
List<element> resultats = new ArrayList<>();
try {
JSONObject json = new JSONObject(jsonRaw);
JSONArray items = json.getJSONArray("items");
for (int i = 0; i < items.length(); i++) {
JSONObject repo = items.getJSONObject(i);
String titre = repo.getString("full_name");
String url = repo.getString("html_url");
String licence = repo.getJSONObject("license").getString("spdx_id");
resultats.add(new element(titre, url, licence, "github", "code"));
}
} catch (org.json.JSONException e) {
System.out.println("error but continue : " + e.getMessage());
}
return resultats;
}
}
@@ -0,0 +1,139 @@
package org.sillyjirafe.db;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
public class Crawl_StateDB
{
String URL = new String();
String insertSql = "INSERT OR IGNORE INTO crawlstates (bind_name, last_page, rnd, error) VALUES (?, ?, ?, ?)";
Connection connection;
public Crawl_StateDB(String BDstr) throws SQLException
{
URL = BDstr;
connection = DriverManager.getConnection(URL);
}
public void LinkToDB() throws SQLException
{
try (Statement stmt = connection.createStatement())
{
stmt.execute("""
CREATE TABLE IF NOT EXISTS crawlstates (
bind_name TEXT PRIMARY KEY,
last_page INTEGER NOT NULL DEFAULT 1,
rnd TEXT NOT NULL DEFAULT 'a'
error INTEGER NOT NULL DEFAULT 0,
)
""");
}
}
public void AddEllement(String bind, int lastpage) throws SQLException
{
try (PreparedStatement ps = connection.prepareStatement(insertSql))
{
ps.setString(1, bind);
ps.setInt(2, lastpage);
ps.executeUpdate();
System.out.printf("%s is now page %d \n", bind, lastpage);
}
}
public int getPage(String bindName) throws SQLException {
String sql = "SELECT last_page FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getInt("last_page");
} else {
return 1;
}
}
}
public String getRND(String bindName) throws SQLException {
String sql = "SELECT rnd FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getString("rnd");
} else {
return "a";
}
}
}
public int getErrors(String bindName) throws SQLException {
String sql = "SELECT error FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getInt("error");
} else {
return 0;
}
}
}
public void updatePage(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, rnd) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET rnd = rnd + 1
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void newError(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET error = error + 1
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void resetError(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET error = 0
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void updateletter(String bindName) throws SQLException
{
if (getErrors(bindName) >= 10)
{
resetError(bindName);
}
}
public void Close() throws SQLException
{
if (connection != null)
{
connection.close();
}
}
}
@@ -0,0 +1,49 @@
package org.sillyjirafe.linker;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.sillyjirafe.db.Crawl_StateDB;
import org.sillyjirafe.db.ElementsDB;
import org.sillyjirafe.json.ApiKeys;
import org.sillyjirafe.uoapi.*;
public class Main {
public static void main(String[] args) throws Exception
{
Out Out = new Out();
ElementsDB elementDB = new ElementsDB("jdbc:sqlite:elements.db");
Crawl_StateDB crawlstateDB = new Crawl_StateDB("jdbc:sqlite:crawlstates.db");
elementDB.LinkToDB();
crawlstateDB.LinkToDB();
List<String> bindNames = Out.PreOut();
Map<String, Integer> pages = new HashMap<>();
for (String nom : bindNames) {
pages.put(nom, crawlstateDB.getPage(nom));
}
Map<String, String> rnds = new HashMap<>();
for (String nom : bindNames) {
rnds.put(nom, crawlstateDB.getRND(nom));
}
List<element> elements = Out.out(pages, ApiKeys.read(), rnds);
for (int i = 0; i < elements.size(); i++)
{
element el = elements.get(i);
elementDB.AddEllement(el.title(), el.url(), el.licence(), el.source(), el.type());
}
for (String name : bindNames) {
crawlstateDB.updatePage(name);
}
elementDB.Close();
crawlstateDB.Close();
}
}
@@ -0,0 +1,55 @@
package org.sillyjirafe.linker;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.sillyjirafe.db.Crawl_StateDB;
import org.sillyjirafe.db.ElementsDB;
import org.sillyjirafe.json.ApiKeys;
import org.sillyjirafe.uoapi.*;
public class Main {
public static void main(String[] args) throws Exception
{
Out Out = new Out();
ElementsDB elementDB = new ElementsDB("jdbc:sqlite:elements.db");
Crawl_StateDB crawlstateDB = new Crawl_StateDB("jdbc:sqlite:crawlstates.db");
elementDB.LinkToDB();
crawlstateDB.LinkToDB();
Map<String, Boolean> succes = Out.getSucces();
List<String> bindNames = Out.PreOut();
Map<String, Integer> pages = new HashMap<>();
for (String nom : bindNames) {
pages.put(nom, crawlstateDB.getPage(nom));
}
Map<String, String> rnds = new HashMap<>();
for (String nom : bindNames) {
rnds.put(nom, crawlstateDB.getRND(nom));
}
List<element> elements = Out.out(pages, ApiKeys.read(), rnds);
for (int i = 0; i < elements.size(); i++)
{
element el = elements.get(i);
elementDB.AddEllement(el.title(), el.url(), el.licence(), el.source(), el.type());
}
for (String name : bindNames) {
if (Succes.getOrDefault(nom, false)) {
crawlstateDB.updatePage(nom, ...); // succès normal, avance
} else {
crawlstateDB.incrementError(nom); // échec, compte une erreur de plus
}
}
elementDB.Close();
crawlstateDB.Close();
}
}
@@ -0,0 +1,45 @@
package org.sillyjirafe.uoapi.binds;
import java.util.ArrayList;
import java.util.List;
import org.json.JSONArray;
import org.json.JSONObject;
import org.sillyjirafe.uoapi.Api;
import org.sillyjirafe.uoapi.element;
public class GitHubLGPLApi implements Api
{
public String getName()
{
return "GitHubLGPLApi";
}
public String getURL(int page, String apikey, String rnd) {
return "https://api.github.com/search/repositories?q="+ rnd + "&license:lgpl-3.0&page=" + page;
}
public List<element> Parse(String jsonRaw) {
List<element> resultats = new ArrayList<>();
try {
JSONObject json = new JSONObject(jsonRaw);
JSONArray items = json.getJSONArray("items");
for (int i = 0; i < items.length(); i++) {
JSONObject repo = items.getJSONObject(i);
String titre = repo.getString("full_name");
String url = repo.getString("html_url");
String licence = repo.getJSONObject("license").getString("spdx_id");
resultats.add(new element(titre, url, licence, "github", "code"));
}
} catch (org.json.JSONException e) {
System.out.println("error but continue : " + e.getMessage());
}
return resultats;
}
}
@@ -0,0 +1,46 @@
package org.sillyjirafe.uoapi;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.sillyjirafe.uoapi.binds.*;
public class Out {
List<Api> Apis = new ArrayList<>();
public List<String> PreOut()
{
Apis.add(new GitHubMITApi());
Apis.add(new GitHubLGPLApi());
Apis.add(new GitHubApacheApi());
Apis.add(new OpenVerseImagesApi());
Apis.add(new OpenVerseAudiosApi());
Apis.add(new PixabayImagesApi());
List<String> bindnames = new ArrayList<>();
for (Api api : Apis)
{
bindnames.add(api.getName());
}
return bindnames;
}
public List<element> out(Map<String, Integer> bindpages, Map<String, String> ApiKeys, Map<String, String> Rnd) throws Exception
{
List<element> Elements = new ArrayList<>();
for (int i = 0; i < Apis.size(); i++)
{
List<element> currentapi = Caller.CallApi(Apis.get(i), bindpages.get(Apis.get(i).getName()), ApiKeys.get(Apis.get(i).getName()), );
for (int j = 0; j < currentapi.size(); j++)
{
Elements.add(currentapi.get(j));
}
}
return Elements;
}
}
@@ -0,0 +1,43 @@
package org.sillyjirafe.uoapi.binds;
import java.util.ArrayList;
import java.util.List;
import org.json.JSONArray;
import org.json.JSONObject;
import org.sillyjirafe.uoapi.Api;
import org.sillyjirafe.uoapi.element;
public class GitHubApacheApi implements Api
{
public String getName()
{
return "GitHubApacheApi";
}
public String getURL(int page, String apikey, String rnd) {
return "https://api.github.com/search/repositories?q="+ rnd +"&license:apache-2.0&page=" + page;
}
public List<element> Parse(String jsonRaw) {
List<element> resultats = new ArrayList<>();
JSONObject json = new JSONObject(jsonRaw);
JSONArray items = json.getJSONArray("items");
for (int i = 0; i < items.length(); i++) {
JSONObject repo = items.getJSONObject(i);
String titre = repo.getString("full_name");
String url = repo.getString("html_url");
String licence = repo.getJSONObject("license").getString("spdx_id");
resultats.add(new element(titre, url, licence, "github", "code"));
}
} catch (org.json.JSONException e) {
System.out.println("error but continue : " + e.getMessage());
}
return resultats;
}
}
@@ -0,0 +1,54 @@
package org.sillyjirafe.linker;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.sillyjirafe.db.Crawl_StateDB;
import org.sillyjirafe.db.ElementsDB;
import org.sillyjirafe.json.ApiKeys;
import org.sillyjirafe.uoapi.*;
public class Main {
public static void main(String[] args) throws Exception
{
Out Out = new Out();
ElementsDB elementDB = new ElementsDB("jdbc:sqlite:elements.db");
Crawl_StateDB crawlstateDB = new Crawl_StateDB("jdbc:sqlite:crawlstates.db");
elementDB.LinkToDB();
crawlstateDB.LinkToDB();
Map<String, Boolean> succes = Out.getSucces();
List<String> bindNames = Out.PreOut();
Map<String, Integer> pages = new HashMap<>();
for (String nom : bindNames) {
pages.put(nom, crawlstateDB.getPage(nom));
}
Map<String, String> rnds = new HashMap<>();
for (String nom : bindNames) {
rnds.put(nom, crawlstateDB.getRND(nom));
}
List<element> elements = Out.out(pages, ApiKeys.read(), rnds);
for (int i = 0; i < elements.size(); i++)
{
element el = elements.get(i);
elementDB.AddEllement(el.title(), el.url(), el.licence(), el.source(), el.type());
}
for (String name : bindNames) {
if (Succes.getOrDefault(nom, false)) {
crawlstateDB.updatePage(nom, ...); // succès normal, avance
} else {
crawlstateDB.incrementError(nom); // échec, compte une erreur de plus
}
}
elementDB.Close();
crawlstateDB.Close();
}
}
@@ -0,0 +1,43 @@
package org.sillyjirafe.uoapi.binds;
import java.util.ArrayList;
import java.util.List;
import org.json.JSONArray;
import org.json.JSONObject;
import org.sillyjirafe.uoapi.Api;
import org.sillyjirafe.uoapi.element;
public class OpenVerseAudiosApi implements Api
{
public String getName()
{
return "OpenVerseAudiosApi";
}
public String getURL(int page, String apikey) {
return "https://api.openverse.org/v1/audio/?format=json&page=" + page;
}
public List<element> Parse(String jsonRaw) {
List<element> resultats = new ArrayList<>();
try {
JSONObject json = new JSONObject(jsonRaw);
JSONArray items = json.getJSONArray("results");
for (int i = 0; i < items.length(); i++) {
JSONObject audio = items.getJSONObject(i);
String titre = audio.getString("title");
String url = audio.getString("url");
String licence = audio.getString("license");
resultats.add(new element(titre, url, licence, "openverse", "audio"));
}
} catch (org.json.JSONException e) {
System.out.println("error but continue : " + e.getMessage());
}
return resultats;
}
}
@@ -0,0 +1,98 @@
package org.sillyjirafe.db;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
public class Crawl_StateDB
{
String URL = new String();
String insertSql = "INSERT OR IGNORE INTO crawlstates (bind_name, last_page, rnd, error) VALUES (?, ?, ?)";
Connection connection;
public Crawl_StateDB(String BDstr) throws SQLException
{
URL = BDstr;
connection = DriverManager.getConnection(URL);
}
public void LinkToDB() throws SQLException
{
try (Statement stmt = connection.createStatement())
{
stmt.execute("""
CREATE TABLE IF NOT EXISTS crawlstates (
bind_name TEXT PRIMARY KEY,
last_page INTEGER NOT NULL DEFAULT 1,
rnd TEXT NOT NULL DEFAULT 'a'
)
""");
}
}
public void AddEllement(String bind, int lastpage) throws SQLException
{
try (PreparedStatement ps = connection.prepareStatement(insertSql))
{
ps.setString(1, bind);
ps.setInt(2, lastpage);
ps.executeUpdate();
System.out.printf("%s is now page %d \n", bind, lastpage);
}
}
public int getPage(String bindName) throws SQLException {
String sql = "SELECT last_page FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getInt("last_page");
} else {
return 1;
}
}
}
public String getRND(String bindName) throws SQLException {
String sql = "SELECT rnd FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getString("rnd");
} else {
return "a";
}
}
}
public void updatePage(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET last_page = last_page + 1
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void updateletter(String bindName)
{
}
public void Close() throws SQLException
{
if (connection != null)
{
connection.close();
}
}
}
@@ -0,0 +1,124 @@
package org.sillyjirafe.db;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
public class Crawl_StateDB
{
String URL = new String();
String insertSql = "INSERT OR IGNORE INTO crawlstates (bind_name, last_page, rnd, error) VALUES (?, ?, ?, ?)";
Connection connection;
public Crawl_StateDB(String BDstr) throws SQLException
{
URL = BDstr;
connection = DriverManager.getConnection(URL);
}
public void LinkToDB() throws SQLException
{
try (Statement stmt = connection.createStatement())
{
stmt.execute("""
CREATE TABLE IF NOT EXISTS crawlstates (
bind_name TEXT PRIMARY KEY,
last_page INTEGER NOT NULL DEFAULT 1,
rnd TEXT NOT NULL DEFAULT 'a'
error INTEGER NOT NULL DEFAULT 0,
)
""");
}
}
public void AddEllement(String bind, int lastpage) throws SQLException
{
try (PreparedStatement ps = connection.prepareStatement(insertSql))
{
ps.setString(1, bind);
ps.setInt(2, lastpage);
ps.executeUpdate();
System.out.printf("%s is now page %d \n", bind, lastpage);
}
}
public int getPage(String bindName) throws SQLException {
String sql = "SELECT last_page FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getInt("last_page");
} else {
return 1;
}
}
}
public String getRND(String bindName) throws SQLException {
String sql = "SELECT rnd FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getString("rnd");
} else {
return "a";
}
}
}
public int getErrors(String bindName) throws SQLException {
String sql = "SELECT error FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getInt("error");
} else {
return 0;
}
}
}
public void updatePage(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, rnd) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET rnd = rnd + 1
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void newError(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET error = error + 1
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void updateletter(String bindName)
{
}
public void Close() throws SQLException
{
if (connection != null)
{
connection.close();
}
}
}
@@ -0,0 +1,43 @@
package org.sillyjirafe.uoapi.binds;
import java.util.ArrayList;
import java.util.List;
import org.json.JSONArray;
import org.json.JSONObject;
import org.sillyjirafe.uoapi.Api;
import org.sillyjirafe.uoapi.element;
public class OpenVerseImagesApi implements Api
{
public String getName()
{
return "OpenVerseImagesApi";
}
public String getURL(int page, String apikey, String rnd) {
return "https://api.openverse.org/v1/images/?q=\n" + rnd+"format=json&page=" + page;
}
public List<element> Parse(String jsonRaw) {
List<element> resultats = new ArrayList<>();
try {
JSONObject json = new JSONObject(jsonRaw);
JSONArray items = json.getJSONArray("results");
for (int i = 0; i < items.length(); i++) {
JSONObject image = items.getJSONObject(i);
String titre = image.getString("title");
String url = image.getString("url");
String licence = image.getString("license");
resultats.add(new element(titre, url, licence, "openverse", "image"));
}
} catch (org.json.JSONException e) {
System.out.println("error but continue : " + e.getMessage());
}
return resultats;
}
}
@@ -0,0 +1,43 @@
package org.sillyjirafe.uoapi.binds;
import java.util.ArrayList;
import java.util.List;
import org.json.JSONArray;
import org.json.JSONObject;
import org.sillyjirafe.uoapi.Api;
import org.sillyjirafe.uoapi.element;
public class OpenVerseImagesApi implements Api
{
public String getName()
{
return "OpenVerseImagesApi";
}
public String getURL(int page, String apikey, String rnd) {
return "https://api.openverse.org/v1/images/?q=\"" + rnd+"\"format=json&page=" + page;
}
public List<element> Parse(String jsonRaw) {
List<element> resultats = new ArrayList<>();
try {
JSONObject json = new JSONObject(jsonRaw);
JSONArray items = json.getJSONArray("results");
for (int i = 0; i < items.length(); i++) {
JSONObject image = items.getJSONObject(i);
String titre = image.getString("title");
String url = image.getString("url");
String licence = image.getString("license");
resultats.add(new element(titre, url, licence, "openverse", "image"));
}
} catch (org.json.JSONException e) {
System.out.println("error but continue : " + e.getMessage());
}
return resultats;
}
}
@@ -0,0 +1,45 @@
package org.sillyjirafe.uoapi.binds;
import org.json.JSONArray;
import org.json.JSONObject;
import org.sillyjirafe.uoapi.Api;
import org.sillyjirafe.uoapi.element;
import java.util.ArrayList;
import java.util.List;
public class GitHubMITApi implements Api
{
public String getName()
{
return "GitHubMITApi";
}
public String getURL(int page, String apikey, String rnd) {
return "https://api.github.com/search/repositories?q="+ rnd +"&license:mit&page=" + page;
}
public List<element> Parse(String jsonRaw) {
List<element> resultats = new ArrayList<>();
try {
JSONObject json = new JSONObject(jsonRaw);
JSONArray items = json.getJSONArray("items");
for (int i = 0; i < items.length(); i++) {
JSONObject repo = items.getJSONObject(i);
String titre = repo.getString("full_name");
String url = repo.getString("html_url");
String licence = repo.getJSONObject("license").getString("spdx_id");
resultats.add(new element(titre, url, licence, "github", "code"));
}
} catch (org.json.JSONException e) {
System.out.println("error but continue : " + e.getMessage());
}
return resultats;
}
}
@@ -0,0 +1,110 @@
package org.sillyjirafe.db;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
public class Crawl_StateDB
{
String URL = new String();
String insertSql = "INSERT OR IGNORE INTO crawlstates (bind_name, last_page, rnd, error) VALUES (?, ?, ?, ?)";
Connection connection;
public Crawl_StateDB(String BDstr) throws SQLException
{
URL = BDstr;
connection = DriverManager.getConnection(URL);
}
public void LinkToDB() throws SQLException
{
try (Statement stmt = connection.createStatement())
{
stmt.execute("""
CREATE TABLE IF NOT EXISTS crawlstates (
bind_name TEXT PRIMARY KEY,
last_page INTEGER NOT NULL DEFAULT 1,
rnd TEXT NOT NULL DEFAULT 'a'
error INTEGER NOT NULL DEFAULT 0,
)
""");
}
}
public void AddEllement(String bind, int lastpage) throws SQLException
{
try (PreparedStatement ps = connection.prepareStatement(insertSql))
{
ps.setString(1, bind);
ps.setInt(2, lastpage);
ps.executeUpdate();
System.out.printf("%s is now page %d \n", bind, lastpage);
}
}
public int getPage(String bindName) throws SQLException {
String sql = "SELECT last_page FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getInt("last_page");
} else {
return 1;
}
}
}
public String getRND(String bindName) throws SQLException {
String sql = "SELECT rnd FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getString("rnd");
} else {
return "a";
}
}
}
public void updatePage(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, rnd) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET rnd = rnd + 1
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void newError(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET last_page = last_page + 1
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void updateletter(String bindName)
{
}
public void Close() throws SQLException
{
if (connection != null)
{
connection.close();
}
}
}
@@ -0,0 +1,44 @@
package org.sillyjirafe.uoapi.binds;
import java.util.ArrayList;
import java.util.List;
import org.json.JSONArray;
import org.json.JSONObject;
import org.sillyjirafe.uoapi.Api;
import org.sillyjirafe.uoapi.element;
public class GitHubApacheApi implements Api
{
public String getName()
{
return "GitHubApacheApi";
}
public String getURL(int page, String apikey) {
return "https://api.github.com/search/repositories?q=license:apache-2.0&page=" + page;
}
public List<element> Parse(String jsonRaw) {
List<element> resultats = new ArrayList<>();
try {
JSONObject json = new JSONObject(jsonRaw);
JSONArray items = json.getJSONArray("items");
for (int i = 0; i < items.length(); i++) {
JSONObject repo = items.getJSONObject(i);
String titre = repo.getString("full_name");
String url = repo.getString("html_url");
String licence = repo.getJSONObject("license").getString("spdx_id");
resultats.add(new element(titre, url, licence, "github", "code"));
}
} catch (org.json.JSONException e) {
System.out.println("error but continue : " + e.getMessage());
}
return resultats;
}
}
@@ -0,0 +1,99 @@
package org.sillyjirafe.db;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
public class Crawl_StateDB
{
String URL = new String();
String insertSql = "INSERT OR IGNORE INTO crawlstates (bind_name, last_page, rnd, error) VALUES (?, ?, ?, ?)";
Connection connection;
public Crawl_StateDB(String BDstr) throws SQLException
{
URL = BDstr;
connection = DriverManager.getConnection(URL);
}
public void LinkToDB() throws SQLException
{
try (Statement stmt = connection.createStatement())
{
stmt.execute("""
CREATE TABLE IF NOT EXISTS crawlstates (
bind_name TEXT PRIMARY KEY,
last_page INTEGER NOT NULL DEFAULT 1,
rnd TEXT NOT NULL DEFAULT 'a'
error INTEGER DEFAULT 0,
)
""");
}
}
public void AddEllement(String bind, int lastpage) throws SQLException
{
try (PreparedStatement ps = connection.prepareStatement(insertSql))
{
ps.setString(1, bind);
ps.setInt(2, lastpage);
ps.executeUpdate();
System.out.printf("%s is now page %d \n", bind, lastpage);
}
}
public int getPage(String bindName) throws SQLException {
String sql = "SELECT last_page FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getInt("last_page");
} else {
return 1;
}
}
}
public String getRND(String bindName) throws SQLException {
String sql = "SELECT rnd FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getString("rnd");
} else {
return "a";
}
}
}
public void updatePage(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET last_page = last_page + 1
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void updateletter(String bindName)
{
}
public void Close() throws SQLException
{
if (connection != null)
{
connection.close();
}
}
}
@@ -0,0 +1,55 @@
package org.sillyjirafe.linker;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.sillyjirafe.db.Crawl_StateDB;
import org.sillyjirafe.db.ElementsDB;
import org.sillyjirafe.json.ApiKeys;
import org.sillyjirafe.uoapi.*;
public class Main {
public static void main(String[] args) throws Exception
{
Out Out = new Out();
ElementsDB elementDB = new ElementsDB("jdbc:sqlite:elements.db");
Crawl_StateDB crawlstateDB = new Crawl_StateDB("jdbc:sqlite:crawlstates.db");
elementDB.LinkToDB();
crawlstateDB.LinkToDB();
Map<String, Boolean> succes = Out.getSucces();
List<String> bindNames = Out.PreOut();
Map<String, Integer> pages = new HashMap<>();
for (String name : bindNames) {
pages.put(name, crawlstateDB.getPage(name));
}
Map<String, String> rnds = new HashMap<>();
for (String nom : bindNames) {
rnds.put(nom, crawlstateDB.getRND(nom));
}
List<element> elements = Out.out(pages, ApiKeys.read(), rnds);
for (int i = 0; i < elements.size(); i++)
{
element el = elements.get(i);
elementDB.AddEllement(el.title(), el.url(), el.licence(), el.source(), el.type());
}
for (String name : bindNames) {
if (succes.getOrDefault(name, false)) {
crawlstateDB.updatePage(name);
} else {
crawlstateDB.newError(name);
}
}
elementDB.Close();
crawlstateDB.Close();
}
}
@@ -0,0 +1,44 @@
package org.sillyjirafe.uoapi.binds;
import java.util.ArrayList;
import java.util.List;
import org.json.JSONArray;
import org.json.JSONObject;
import org.sillyjirafe.uoapi.Api;
import org.sillyjirafe.uoapi.element;
public class GitHubApacheApi implements Api
{
public String getName()
{
return "GitHubApacheApi";
}
public String getURL(int page, String apikey, String rnd) {
return "https://api.github.com/search/repositories?q=license:apache-2.0&page=" + page;
}
public List<element> Parse(String jsonRaw) {
List<element> resultats = new ArrayList<>();
try {
JSONObject json = new JSONObject(jsonRaw);
JSONArray items = json.getJSONArray("items");
for (int i = 0; i < items.length(); i++) {
JSONObject repo = items.getJSONObject(i);
String titre = repo.getString("full_name");
String url = repo.getString("html_url");
String licence = repo.getJSONObject("license").getString("spdx_id");
resultats.add(new element(titre, url, licence, "github", "code"));
}
} catch (org.json.JSONException e) {
System.out.println("error but continue : " + e.getMessage());
}
return resultats;
}
}
@@ -0,0 +1,150 @@
package org.sillyjirafe.db;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
public class Crawl_StateDB
{
String URL = new String();
String insertSql = "INSERT OR IGNORE INTO crawlstates (bind_name, last_page, rnd, error) VALUES (?, ?, ?, ?)";
Connection connection;
public Crawl_StateDB(String BDstr) throws SQLException
{
URL = BDstr;
connection = DriverManager.getConnection(URL);
}
public void LinkToDB() throws SQLException
{
try (Statement stmt = connection.createStatement())
{
stmt.execute("""
CREATE TABLE IF NOT EXISTS crawlstates (
bind_name TEXT PRIMARY KEY,
last_page INTEGER NOT NULL DEFAULT 1,
rnd TEXT NOT NULL DEFAULT 'a'
error INTEGER NOT NULL DEFAULT 0,
)
""");
}
}
public void AddEllement(String bind, int lastpage) throws SQLException
{
try (PreparedStatement ps = connection.prepareStatement(insertSql))
{
ps.setString(1, bind);
ps.setInt(2, lastpage);
ps.executeUpdate();
System.out.printf("%s is now page %d \n", bind, lastpage);
}
}
public int getPage(String bindName) throws SQLException {
String sql = "SELECT last_page FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getInt("last_page");
} else {
return 1;
}
}
}
public String getRND(String bindName) throws SQLException {
String sql = "SELECT rnd FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getString("rnd");
} else {
return "a";
}
}
}
public int getErrors(String bindName) throws SQLException {
String sql = "SELECT error FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getInt("error");
} else {
return 0;
}
}
}
public void updatePage(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET last_page = las_page + 1
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void newError(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET error = error + 1
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void resetError(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET error = 0
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void resetPage(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET last_page = 0
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void updateletter(String bindName) throws SQLException
{
if (getErrors(bindName) >= 10)
{
resetError(bindName);
}
}
public void Close() throws SQLException
{
if (connection != null)
{
connection.close();
}
}
}
@@ -0,0 +1,124 @@
package org.sillyjirafe.db;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
public class Crawl_StateDB
{
String URL = new String();
String insertSql = "INSERT OR IGNORE INTO crawlstates (bind_name, last_page, rnd, error) VALUES (?, ?, ?, ?)";
Connection connection;
public Crawl_StateDB(String BDstr) throws SQLException
{
URL = BDstr;
connection = DriverManager.getConnection(URL);
}
public void LinkToDB() throws SQLException
{
try (Statement stmt = connection.createStatement())
{
stmt.execute("""
CREATE TABLE IF NOT EXISTS crawlstates (
bind_name TEXT PRIMARY KEY,
last_page INTEGER NOT NULL DEFAULT 1,
rnd TEXT NOT NULL DEFAULT 'a'
error INTEGER NOT NULL DEFAULT 0,
)
""");
}
}
public void AddEllement(String bind, int lastpage) throws SQLException
{
try (PreparedStatement ps = connection.prepareStatement(insertSql))
{
ps.setString(1, bind);
ps.setInt(2, lastpage);
ps.executeUpdate();
System.out.printf("%s is now page %d \n", bind, lastpage);
}
}
public int getPage(String bindName) throws SQLException {
String sql = "SELECT last_page FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getInt("last_page");
} else {
return 1;
}
}
}
public String getRND(String bindName) throws SQLException {
String sql = "SELECT rnd FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getString("rnd");
} else {
return "a";
}
}
}
public int getErrors(String bindName) throws SQLException {
String sql = "SELECT error FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getInt("error");
} else {
return 0;
}
}
}
public void updatePage(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, rnd) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET rnd = rnd + 1
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void newError(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET last_page = last_page + 1
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void updateletter(String bindName)
{
}
public void Close() throws SQLException
{
if (connection != null)
{
connection.close();
}
}
}
@@ -0,0 +1,150 @@
package org.sillyjirafe.db;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
public class Crawl_StateDB
{
String URL = new String();
String insertSql = "INSERT OR IGNORE INTO crawlstates (bind_name, last_page, rnd, error) VALUES (?, ?, ?, ?)";
Connection connection;
public Crawl_StateDB(String BDstr) throws SQLException
{
URL = BDstr;
connection = DriverManager.getConnection(URL);
}
public void LinkToDB() throws SQLException
{
try (Statement stmt = connection.createStatement())
{
stmt.execute("""
CREATE TABLE IF NOT EXISTS crawlstates (
bind_name TEXT PRIMARY KEY,
last_page INTEGER NOT NULL DEFAULT 1,
rnd TEXT NOT NULL DEFAULT 'a'
error INTEGER NOT NULL DEFAULT 0,
)
""");
}
}
public void AddEllement(String bind, int lastpage) throws SQLException
{
try (PreparedStatement ps = connection.prepareStatement(insertSql))
{
ps.setString(1, bind);
ps.setInt(2, lastpage);
ps.executeUpdate();
System.out.printf("%s is now page %d \n", bind, lastpage);
}
}
public int getPage(String bindName) throws SQLException {
String sql = "SELECT last_page FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getInt("last_page");
} else {
return 1;
}
}
}
public String getRND(String bindName) throws SQLException {
String sql = "SELECT rnd FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getString("rnd");
} else {
return "a";
}
}
}
public int getErrors(String bindName) throws SQLException {
String sql = "SELECT error FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getInt("error");
} else {
return 0;
}
}
}
public void updatePage(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET last_page = las_page + 1
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void newError(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET error = error + 1
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void resetError(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET error = 0
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void resetPage(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET last_page = 0
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void updateletter(String bindName) throws SQLException
{
if (getErrors(bindName) >= 10)
{
resetError(bindName);
resetPage(bindName);
}
}
public void Close() throws SQLException
{
if (connection != null)
{
connection.close();
}
}
}
@@ -0,0 +1,43 @@
package org.sillyjirafe.uoapi.binds;
import java.util.ArrayList;
import java.util.List;
import org.json.JSONArray;
import org.json.JSONObject;
import org.sillyjirafe.uoapi.Api;
import org.sillyjirafe.uoapi.element;
public class OpenVerseAudiosApi implements Api
{
public String getName()
{
return "OpenVerseAudiosApi";
}
public String getURL(int page, String apikey, String rnd) {
return "https://api.openverse.org/v1/audio/?q="+ rnd +"format=json&page=" + page;
}
public List<element> Parse(String jsonRaw) {
List<element> resultats = new ArrayList<>();
try {
JSONObject json = new JSONObject(jsonRaw);
JSONArray items = json.getJSONArray("results");
for (int i = 0; i < items.length(); i++) {
JSONObject audio = items.getJSONObject(i);
String titre = audio.getString("title");
String url = audio.getString("url");
String licence = audio.getString("license");
resultats.add(new element(titre, url, licence, "openverse", "audio"));
}
} catch (org.json.JSONException e) {
System.out.println("error but continue : " + e.getMessage());
}
return resultats;
}
}
@@ -0,0 +1,139 @@
package org.sillyjirafe.db;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
public class Crawl_StateDB
{
String URL = new String();
String insertSql = "INSERT OR IGNORE INTO crawlstates (bind_name, last_page, rnd, error) VALUES (?, ?, ?, ?)";
Connection connection;
public Crawl_StateDB(String BDstr) throws SQLException
{
URL = BDstr;
connection = DriverManager.getConnection(URL);
}
public void LinkToDB() throws SQLException
{
try (Statement stmt = connection.createStatement())
{
stmt.execute("""
CREATE TABLE IF NOT EXISTS crawlstates (
bind_name TEXT PRIMARY KEY,
last_page INTEGER NOT NULL DEFAULT 1,
rnd TEXT NOT NULL DEFAULT 'a'
error INTEGER NOT NULL DEFAULT 0,
)
""");
}
}
public void AddEllement(String bind, int lastpage) throws SQLException
{
try (PreparedStatement ps = connection.prepareStatement(insertSql))
{
ps.setString(1, bind);
ps.setInt(2, lastpage);
ps.executeUpdate();
System.out.printf("%s is now page %d \n", bind, lastpage);
}
}
public int getPage(String bindName) throws SQLException {
String sql = "SELECT last_page FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getInt("last_page");
} else {
return 1;
}
}
}
public String getRND(String bindName) throws SQLException {
String sql = "SELECT rnd FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getString("rnd");
} else {
return "a";
}
}
}
public int getErrors(String bindName) throws SQLException {
String sql = "SELECT error FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getInt("error");
} else {
return 0;
}
}
}
public void updatePage(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET last_page = las_page + 1
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void newError(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET error = error + 1
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void resetError(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, error) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET error = 0
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void updateletter(String bindName) throws SQLException
{
if (getErrors(bindName) >= 10)
{
resetError(bindName);
}
}
public void Close() throws SQLException
{
if (connection != null)
{
connection.close();
}
}
}
@@ -0,0 +1,44 @@
package org.sillyjirafe.uoapi.binds;
import java.util.ArrayList;
import java.util.List;
import org.json.JSONArray;
import org.json.JSONObject;
import org.sillyjirafe.uoapi.Api;
import org.sillyjirafe.uoapi.element;
public class PixabayImagesApi implements Api
{
public String getName()
{
return "PixabayImagesApi";
}
public String getURL(int page, String apikey) {
return "https://pixabay.com/api/?per_page=200&page=" + page + "&key=" + apikey;
}
public List<element> Parse(String jsonRaw) {
List<element> resultats = new ArrayList<>();
try {
JSONObject json = new JSONObject(jsonRaw);
JSONArray items = json.getJSONArray("hits");
for (int i = 0; i < items.length(); i++) {
JSONObject repo = items.getJSONObject(i);
String titre = repo.getString("name");
String url = repo.getString("pageURL");
String licence = "cc0";
resultats.add(new element(titre, url, licence, "pixabay", "image"));
}
} catch (org.json.JSONException e) {
System.out.println("error but continue : " + e.getMessage());
}
return resultats;
}
}
@@ -0,0 +1,55 @@
package org.sillyjirafe.linker;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.sillyjirafe.db.Crawl_StateDB;
import org.sillyjirafe.db.ElementsDB;
import org.sillyjirafe.json.ApiKeys;
import org.sillyjirafe.uoapi.*;
public class Main {
public static void main(String[] args) throws Exception
{
Out Out = new Out();
ElementsDB elementDB = new ElementsDB("jdbc:sqlite:elements.db");
Crawl_StateDB crawlstateDB = new Crawl_StateDB("jdbc:sqlite:crawlstates.db");
elementDB.LinkToDB();
crawlstateDB.LinkToDB();
Map<String, Boolean> succes = Out.getSucces();
List<String> bindNames = Out.PreOut();
Map<String, Integer> pages = new HashMap<>();
for (String name : bindNames) {
pages.put(name, crawlstateDB.getPage(name));
}
Map<String, String> rnds = new HashMap<>();
for (String nom : bindNames) {
rnds.put(nom, crawlstateDB.getRND(nom));
}
List<element> elements = Out.out(pages, ApiKeys.read(), rnds);
for (int i = 0; i < elements.size(); i++)
{
element el = elements.get(i);
elementDB.AddEllement(el.title(), el.url(), el.licence(), el.source(), el.type());
}
for (String name : bindNames) {
if (succes.getOrDefault(name, false)) {
crawlstateDB.updatePage(name);
} else {
crawlstateDB.incrementError(name); // échec, compte une erreur de plus
}
}
elementDB.Close();
crawlstateDB.Close();
}
}
@@ -0,0 +1,124 @@
package org.sillyjirafe.db;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
public class Crawl_StateDB
{
String URL = new String();
String insertSql = "INSERT OR IGNORE INTO crawlstates (bind_name, last_page, rnd, error) VALUES (?, ?, ?, ?)";
Connection connection;
public Crawl_StateDB(String BDstr) throws SQLException
{
URL = BDstr;
connection = DriverManager.getConnection(URL);
}
public void LinkToDB() throws SQLException
{
try (Statement stmt = connection.createStatement())
{
stmt.execute("""
CREATE TABLE IF NOT EXISTS crawlstates (
bind_name TEXT PRIMARY KEY,
last_page INTEGER NOT NULL DEFAULT 1,
rnd TEXT NOT NULL DEFAULT 'a'
error INTEGER NOT NULL DEFAULT 0,
)
""");
}
}
public void AddEllement(String bind, int lastpage) throws SQLException
{
try (PreparedStatement ps = connection.prepareStatement(insertSql))
{
ps.setString(1, bind);
ps.setInt(2, lastpage);
ps.executeUpdate();
System.out.printf("%s is now page %d \n", bind, lastpage);
}
}
public int getPage(String bindName) throws SQLException {
String sql = "SELECT last_page FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getInt("last_page");
} else {
return 1;
}
}
}
public String getRND(String bindName) throws SQLException {
String sql = "SELECT rnd FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getString("rnd");
} else {
return "a";
}
}
}
public int getErrors(String bindName) throws SQLException {
String sql = "SELECT error FROM crawlstates WHERE bind_name = ?";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ResultSet rs = ps.executeQuery();
if (rs.next()) {
return rs.getInt("error");
} else {
return 1;
}
}
}
public void updatePage(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, rnd) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET rnd = rnd + 1
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void newError(String bindName) throws SQLException {
String sql = """
INSERT INTO crawlstates (bind_name, last_page) VALUES (?, 2)
ON CONFLICT(bind_name) DO UPDATE SET last_page = last_page + 1
""";
try (PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, bindName);
ps.executeUpdate();
}
}
public void updateletter(String bindName)
{
}
public void Close() throws SQLException
{
if (connection != null)
{
connection.close();
}
}
}
@@ -0,0 +1,46 @@
package org.sillyjirafe.uoapi;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.sillyjirafe.uoapi.binds.*;
public class Out {
List<Api> Apis = new ArrayList<>();
public List<String> PreOut()
{
Apis.add(new GitHubMITApi());
Apis.add(new GitHubLGPLApi());
Apis.add(new GitHubApacheApi());
Apis.add(new OpenVerseImagesApi());
Apis.add(new OpenVerseAudiosApi());
Apis.add(new PixabayImagesApi());
List<String> bindnames = new ArrayList<>();
for (Api api : Apis)
{
bindnames.add(api.getName());
}
return bindnames;
}
public List<element> out(Map<String, Integer> bindpages, Map<String, String> ApiKeys, Map<String, String> Rnds) throws Exception
{
List<element> Elements = new ArrayList<>();
for (int i = 0; i < Apis.size(); i++)
{
List<element> currentapi = Caller.CallApi(Apis.get(i), bindpages.get(Apis.get(i).getName()), ApiKeys.get(Apis.get(i).getName()), Rnds.get(Apis.get(i).getName()));
for (int j = 0; j < currentapi.size(); j++)
{
Elements.add(currentapi.get(j));
}
}
return Elements;
}
}
@@ -1,2 +1,2 @@
eclipse.preferences.version=1
org.eclipse.jdt.launching.PREF_VM_XML=<?xml version\="1.0" encoding\="UTF-8" standalone\="no"?>\n<vmSettings defaultVM\="57,org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType13,1785979565653" defaultVMConnector\="">\n <vmType id\="org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType">\n <vm id\="1785979565653" name\="jre" path\="/app/eclipse/plugins/org.eclipse.justj.openjdk.hotspot.jre.full.linux.x86_64_21.0.11.v20260515-1531/jre"/>\n </vmType>\n</vmSettings>\n
org.eclipse.jdt.launching.PREF_VM_XML=<?xml version\="1.0" encoding\="UTF-8" standalone\="no"?>\n<vmSettings defaultVM\="57,org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType13,1791126493219" defaultVMConnector\="">\n <vmType id\="org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType">\n <vm id\="1791126493219" name\="jre" path\="/app/eclipse/plugins/org.eclipse.justj.openjdk.hotspot.jre.full.linux.x86_64_25.0.4.v20260826-0822/jre"/>\n </vmType>\n</vmSettings>\n
@@ -0,0 +1,2 @@
eclipse.preferences.version=1
org.eclipse.ui.console.zoomFontHeights=org.eclipse.debug.ui.ProcessConsoleType\=10|0
@@ -1,6 +1,6 @@
IMPORT_FILES_AND_FOLDERS_RELATIVE=true
IMPORT_FILES_AND_FOLDERS_TYPE=23,1
eclipse.preferences.version=1
platformState=945730117414574
platformState=943949841082661
quickStart=false
tipsAndTricks=true
@@ -6,6 +6,7 @@
<listAttribute key="org.eclipse.debug.core.MAPPED_RESOURCE_TYPES">
<listEntry value="1"/>
</listAttribute>
<stringAttribute key="org.eclipse.debug.core.terminate.timestamp" value="1791670329354"/>
<booleanAttribute key="org.eclipse.jdt.launching.ATTR_EXCLUDE_TEST_CODE" value="true"/>
<stringAttribute key="org.eclipse.jdt.launching.MAIN_TYPE" value="org.sillyjirafe.linker.Main"/>
<stringAttribute key="org.eclipse.jdt.launching.MODULE_NAME" value="SillyBlahaj.org"/>
@@ -10,16 +10,16 @@
<mruHistory/>
<favorites/>
</launchGroup>
<launchGroup id="org.eclipse.ui.externaltools.launchGroup">
<mruHistory/>
<favorites/>
</launchGroup>
<launchGroup id="org.eclipse.eclemma.ui.launchGroup.coverage">
<mruHistory>
<launch memento="&lt;?xml version=&quot;1.0&quot; encoding=&quot;UTF-8&quot; standalone=&quot;no&quot;?&gt;&#10;&lt;launchConfiguration local=&quot;true&quot; path=&quot;Main (3)&quot;/&gt;&#10;"/>
</mruHistory>
<favorites/>
</launchGroup>
<launchGroup id="org.eclipse.ui.externaltools.launchGroup">
<mruHistory/>
<favorites/>
</launchGroup>
<launchGroup id="org.eclipse.debug.ui.launchGroup.run">
<mruHistory>
<launch memento="&lt;?xml version=&quot;1.0&quot; encoding=&quot;UTF-8&quot; standalone=&quot;no&quot;?&gt;&#10;&lt;launchConfiguration local=&quot;true&quot; path=&quot;Main (3)&quot;/&gt;&#10;"/>
@@ -0,0 +1,3 @@
#Eclipse last used NL locale
#Sat Oct 10 23:02:27 CEST 2026
eclipse.last.nl=en_US
File diff suppressed because it is too large. Load diff
@@ -1,7 +1,9 @@
INDEX VERSION 1.134+/home/kotama/Silly-Jirafe/.metadata/.plugins/org.eclipse.jdt.core
1568060524.index
4203645977.index
1865797976.index
206920101.index
INDEX VERSION 1.134+/home/Kotama_Chio/Musique/SillyBlahaj.org/Java/.metadata/.plugins/org.eclipse.jdt.core
263131634.index
32071656.index
2407587206.index
2916335905.index
3079814413.index
3263143359.index
206920101.index
2910310629.index
1865797976.index
@@ -2,4 +2,6 @@
<dirs>
<entry loc="/app/eclipse/plugins/org.eclipse.justj.openjdk.hotspot.jre.full.linux.x86_64_21.0.11.v20260515-1531/jre" stamp="0"/>
<entry loc="/app/eclipse/plugins/org.eclipse.justj.openjdk.hotspot.jre.full.linux.x86_64_21.0.11.v20260515-1531" stamp="0"/>
<entry loc="/app/eclipse/plugins/org.eclipse.justj.openjdk.hotspot.jre.full.linux.x86_64_25.0.4.v20260826-0822" stamp="0"/>
<entry loc="/app/eclipse/plugins/org.eclipse.justj.openjdk.hotspot.jre.full.linux.x86_64_25.0.4.v20260826-0822/jre" stamp="0"/>
</dirs>
@@ -12,4 +12,16 @@
<entry path="null"/>
</endorsedDirs>
</libraryInfo>
<libraryInfo home="/app/eclipse/plugins/org.eclipse.justj.openjdk.hotspot.jre.full.linux.x86_64_25.0.4.v20260826-0822" version="25.0.4.1">
<bootpath>
<entry path="null"/>
</bootpath>
<extensionDirs>
<entry path="null"/>
</extensionDirs>
<endorsedDirs>
<entry path="null"/>
</endorsedDirs>
</libraryInfo>
<libraryInfo home="/app/eclipse/plugins/org.eclipse.justj.openjdk.hotspot.jre.full.linux.x86_64_25.0.4.v20260826-0822/jre" version="25.0.4.1"/>
</libraryInfos>
@@ -2,13 +2,13 @@
<section name="Workbench">
<item key="org.eclipse.jdt.ui.last.selected.jre.kind2" value="2"/>
<item key="org.eclipse.jdt.ui.last.selected.execution.enviroment" value="JavaSE-21"/>
<item key="org.eclipse.jdt.ui.lastextjar" value="/home/kotama/Silly-Jirafe/libs"/>
<item key="org.eclipse.jdt.ui.lastextjar" value="/home/Kotama_Chio/Musique/SillyBlahaj.org/Java/libs"/>
<section name="org.eclipse.jdt.internal.ui.packageview.PackageExplorerPart">
<item key="group_libraries" value="true"/>
<item key="layout" value="2"/>
<item key="rootMode" value="1"/>
<item key="linkWithEditor" value="false"/>
<item key="memento" value="&lt;?xml version=&quot;1.0&quot; encoding=&quot;UTF-8&quot;?&gt;&#x0A;&lt;packageExplorer group_libraries=&quot;1&quot; layout=&quot;2&quot; linkWithEditor=&quot;0&quot; rootMode=&quot;1&quot; workingSetName=&quot;Aggregate for window 1785979556746&quot;&gt;&#x0A;&lt;customFilters userDefinedPatternsEnabled=&quot;false&quot;&gt;&#x0A;&lt;xmlDefinedFilters&gt;&#x0A;&lt;child filterId=&quot;org.eclipse.jdt.ui.PackageExplorer.StaticsFilter&quot; isEnabled=&quot;false&quot;/&gt;&#x0A;&lt;child filterId=&quot;org.eclipse.buildship.ui.packageexplorer.filter.gradle.buildfolder&quot; isEnabled=&quot;true&quot;/&gt;&#x0A;&lt;child filterId=&quot;org.eclipse.mylyn.java.ui.MembersFilter&quot; isEnabled=&quot;false&quot;/&gt;&#x0A;&lt;child filterId=&quot;org.eclipse.jdt.ui.PackageExplorer.NonJavaProjectsFilter&quot; isEnabled=&quot;false&quot;/&gt;&#x0A;&lt;child filterId=&quot;org.eclipse.jdt.ui.PackageExplorer_patternFilterId_.*&quot; isEnabled=&quot;false&quot;/&gt;&#x0A;&lt;child filterId=&quot;org.eclipse.jdt.ui.PackageExplorer.NonSharedProjectsFilter&quot; isEnabled=&quot;false&quot;/&gt;&#x0A;&lt;child filterId=&quot;org.eclipse.jdt.ui.PackageExplorer.SyntheticMembersFilter&quot; isEnabled=&quot;true&quot;/&gt;&#x0A;&lt;child filterId=&quot;org.eclipse.jdt.ui.PackageExplorer.ContainedLibraryFilter&quot; isEnabled=&quot;false&quot;/&gt;&#x0A;&lt;child filterId=&quot;org.eclipse.jdt.internal.ui.PackageExplorer.HideInnerClassFilesFilter&quot; isEnabled=&quot;true&quot;/&gt;&#x0A;&lt;child filterId=&quot;org.eclipse.jdt.internal.ui.PackageExplorer.EmptyInnerPackageFilter&quot; isEnabled=&quot;true&quot;/&gt;&#x0A;&lt;child filterId=&quot;org.eclipse.m2e.MavenModuleFilter&quot; isEnabled=&quot;false&quot;/&gt;&#x0A;&lt;child filterId=&quot;org.eclipse.buildship.ui.packageexplorer.filter.gradle.subProject&quot; isEnabled=&quot;true&quot;/&gt;&#x0A;&lt;child filterId=&quot;org.eclipse.jdt.ui.PackageExplorer.ClosedProjectsFilter&quot; isEnabled=&quot;false&quot;/&gt;&#x0A;&lt;child filterId=&quot;org.eclipse.jdt.ui.PackageExplorer.DeprecatedMembersFilter&quot; isEnabled=&quot;false&quot;/&gt;&#x0A;&lt;child filterId=&quot;org.eclipse.jdt.ui.PackageExplorer.EmptyLibraryContainerFilter&quot; isEnabled=&quot;true&quot;/&gt;&#x0A;&lt;child filterId=&quot;org.eclipse.jdt.ui.PackageExplorer.PackageDeclarationFilter&quot; isEnabled=&quot;true&quot;/&gt;&#x0A;&lt;child filterId=&quot;org.eclipse.jdt.ui.PackageExplorer.ImportDeclarationFilter&quot; isEnabled=&quot;true&quot;/&gt;&#x0A;&lt;child filterId=&quot;org.eclipse.jdt.ui.PackageExplorer.NonJavaElementFilter&quot; isEnabled=&quot;false&quot;/&gt;&#x0A;&lt;child filterId=&quot;org.eclipse.jdt.ui.PackageExplorer.LibraryFilter&quot; isEnabled=&quot;false&quot;/&gt;&#x0A;&lt;child filterId=&quot;org.eclipse.jdt.ui.PackageExplorer.CuAndClassFileFilter&quot; isEnabled=&quot;false&quot;/&gt;&#x0A;&lt;child filterId=&quot;org.eclipse.jdt.internal.ui.PackageExplorer.EmptyPackageFilter&quot; isEnabled=&quot;false&quot;/&gt;&#x0A;&lt;child filterId=&quot;org.eclipse.jdt.ui.PackageExplorer.NonPublicFilter&quot; isEnabled=&quot;false&quot;/&gt;&#x0A;&lt;child filterId=&quot;org.eclipse.jdt.ui.PackageExplorer.LocalTypesFilter&quot; isEnabled=&quot;false&quot;/&gt;&#x0A;&lt;child filterId=&quot;org.eclipse.jdt.ui.PackageExplorer.FieldsFilter&quot; isEnabled=&quot;false&quot;/&gt;&#x0A;&lt;/xmlDefinedFilters&gt;&#x0A;&lt;/customFilters&gt;&#x0A;&lt;/packageExplorer&gt;"/>
<item key="memento" value="&lt;?xml version=&quot;1.0&quot; encoding=&quot;UTF-8&quot;?&gt;&#x0A;&lt;packageExplorer group_libraries=&quot;1&quot; layout=&quot;2&quot; linkWithEditor=&quot;0&quot; rootMode=&quot;1&quot; workingSetName=&quot;Aggregate for window 1785979556746&quot;&gt;&#x0A;&lt;customFilters userDefinedPatternsEnabled=&quot;false&quot;&gt;&#x0A;&lt;xmlDefinedFilters&gt;&#x0A;&lt;child filterId=&quot;org.eclipse.jdt.ui.PackageExplorer.StaticsFilter&quot; isEnabled=&quot;false&quot;/&gt;&#x0A;&lt;child filterId=&quot;org.eclipse.buildship.ui.packageexplorer.filter.gradle.buildfolder&quot; isEnabled=&quot;true&quot;/&gt;&#x0A;&lt;child filterId=&quot;org.eclipse.mylyn.java.ui.MembersFilter&quot; isEnabled=&quot;false&quot;/&gt;&#x0A;&lt;child filterId=&quot;org.eclipse.jdt.ui.PackageExplorer.NonJavaProjectsFilter&quot; isEnabled=&quot;false&quot;/&gt;&#x0A;&lt;child filterId=&quot;org.eclipse.jdt.ui.PackageExplorer_patternFilterId_.*&quot; isEnabled=&quot;false&quot;/&gt;&#x0A;&lt;child filterId=&quot;org.eclipse.jdt.ui.PackageExplorer.NonSharedProjectsFilter&quot; isEnabled=&quot;false&quot;/&gt;&#x0A;&lt;child filterId=&quot;org.eclipse.jdt.ui.PackageExplorer.SyntheticMembersFilter&quot; isEnabled=&quot;true&quot;/&gt;&#x0A;&lt;child filterId=&quot;org.eclipse.jdt.ui.PackageExplorer.ContainedLibraryFilter&quot; isEnabled=&quot;false&quot;/&gt;&#x0A;&lt;child filterId=&quot;org.eclipse.m2e.MavenModuleFilter&quot; isEnabled=&quot;false&quot;/&gt;&#x0A;&lt;child filterId=&quot;org.eclipse.jdt.internal.ui.PackageExplorer.HideInnerClassFilesFilter&quot; isEnabled=&quot;true&quot;/&gt;&#x0A;&lt;child filterId=&quot;org.eclipse.jdt.internal.ui.PackageExplorer.EmptyInnerPackageFilter&quot; isEnabled=&quot;true&quot;/&gt;&#x0A;&lt;child filterId=&quot;org.eclipse.buildship.ui.packageexplorer.filter.gradle.subProject&quot; isEnabled=&quot;true&quot;/&gt;&#x0A;&lt;child filterId=&quot;org.eclipse.jdt.ui.PackageExplorer.ClosedProjectsFilter&quot; isEnabled=&quot;false&quot;/&gt;&#x0A;&lt;child filterId=&quot;org.eclipse.jdt.ui.PackageExplorer.DeprecatedMembersFilter&quot; isEnabled=&quot;false&quot;/&gt;&#x0A;&lt;child filterId=&quot;org.eclipse.jdt.ui.PackageExplorer.EmptyLibraryContainerFilter&quot; isEnabled=&quot;true&quot;/&gt;&#x0A;&lt;child filterId=&quot;org.eclipse.jdt.ui.PackageExplorer.PackageDeclarationFilter&quot; isEnabled=&quot;true&quot;/&gt;&#x0A;&lt;child filterId=&quot;org.eclipse.jdt.ui.PackageExplorer.ImportDeclarationFilter&quot; isEnabled=&quot;true&quot;/&gt;&#x0A;&lt;child filterId=&quot;org.eclipse.jdt.ui.PackageExplorer.NonJavaElementFilter&quot; isEnabled=&quot;false&quot;/&gt;&#x0A;&lt;child filterId=&quot;org.eclipse.jdt.ui.PackageExplorer.LibraryFilter&quot; isEnabled=&quot;false&quot;/&gt;&#x0A;&lt;child filterId=&quot;org.eclipse.jdt.ui.PackageExplorer.CuAndClassFileFilter&quot; isEnabled=&quot;false&quot;/&gt;&#x0A;&lt;child filterId=&quot;org.eclipse.jdt.internal.ui.PackageExplorer.EmptyPackageFilter&quot; isEnabled=&quot;false&quot;/&gt;&#x0A;&lt;child filterId=&quot;org.eclipse.jdt.ui.PackageExplorer.NonPublicFilter&quot; isEnabled=&quot;false&quot;/&gt;&#x0A;&lt;child filterId=&quot;org.eclipse.jdt.ui.PackageExplorer.LocalTypesFilter&quot; isEnabled=&quot;false&quot;/&gt;&#x0A;&lt;child filterId=&quot;org.eclipse.jdt.ui.PackageExplorer.FieldsFilter&quot; isEnabled=&quot;false&quot;/&gt;&#x0A;&lt;/xmlDefinedFilters&gt;&#x0A;&lt;/customFilters&gt;&#x0A;&lt;/packageExplorer&gt;"/>
</section>
<section name="JavaProjectWizard.dialogBounds">
<item key="DIALOG_X_ORIGIN" value="0"/>
@@ -0,0 +1,4 @@
<?xml version="1.0" encoding="UTF-8"?>
<session version="1.0">
<refactoring comment="Copy element &apos;libs&apos; to &apos;sillyjirafe.org.api&apos;&#x0A;- Original project: &apos;SillyJirafe.org&apos;&#x0A;- Destination element: &apos;sillyjirafe.org.api&apos;&#x0A;- Original element: &apos;libs&apos;" description="Copy folder" destination="=sillyjirafe.org.api" element1="libs" files="0" flags="589830" folders="1" id="org.eclipse.jdt.ui.copy" policy="org.eclipse.jdt.ui.copyResources" stamp="1791564728729" units="0" version="1.0"/>
</session>
@@ -39,3 +39,11 @@
2026-09-22 16:37:19,751 [Worker-7: Loading available Gradle versions] INFO o.e.b.c.i.u.g.PublishedGradleVersions - Gradle version information cache is out-of-date. Trying to update.
2026-09-25 09:23:41,947 [Worker-6: Loading available Gradle versions] INFO o.e.b.c.i.u.g.PublishedGradleVersions - Gradle version information cache is up-to-date. Trying to read.
2026-09-29 23:22:34,391 [Worker-6: Loading available Gradle versions] INFO o.e.b.c.i.u.g.PublishedGradleVersions - Gradle version information cache is up-to-date. Trying to read.
2026-10-04 17:08:14,297 [Worker-1: Loading available Gradle versions] INFO o.e.b.c.i.u.g.PublishedGradleVersions - Gradle version information cache is up-to-date. Trying to read.
2026-10-04 22:27:35,351 [Worker-1: Loading available Gradle versions] INFO o.e.b.c.i.u.g.PublishedGradleVersions - Gradle version information cache is up-to-date. Trying to read.
2026-10-06 13:16:34,293 [Worker-5: Loading available Gradle versions] INFO o.e.b.c.i.u.g.PublishedGradleVersions - Gradle version information cache is out-of-date. Trying to update.
2026-10-07 00:08:16,911 [Worker-8: Loading available Gradle versions] INFO o.e.b.c.i.u.g.PublishedGradleVersions - Gradle version information cache is up-to-date. Trying to read.
2026-10-07 16:37:42,461 [Worker-2: Loading available Gradle versions] INFO o.e.b.c.i.u.g.PublishedGradleVersions - Gradle version information cache is up-to-date. Trying to read.
2026-10-07 21:50:08,486 [Worker-7: Loading available Gradle versions] INFO o.e.b.c.i.u.g.PublishedGradleVersions - Gradle version information cache is up-to-date. Trying to read.
2026-10-09 18:50:09,616 [Worker-9: Loading available Gradle versions] INFO o.e.b.c.i.u.g.PublishedGradleVersions - Gradle version information cache is up-to-date. Trying to read.
2026-10-10 23:02:35,211 [Worker-7: Loading available Gradle versions] INFO o.e.b.c.i.u.g.PublishedGradleVersions - Gradle version information cache is out-of-date. Trying to update.
@@ -0,0 +1,41 @@
<configuration scan="true">
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
<encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
<pattern>%date [%thread] %-5level %logger{35} - %msg%n</pattern>
</encoder>
<filter class="ch.qos.logback.classic.filter.ThresholdFilter">
<level>${org.eclipse.m2e.log.console.threshold:-OFF}</level> <!-- change to DEBUG to mimic '-consolelog' behaviour -->
</filter>
</appender>
<appender name="FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
<File>${org.eclipse.m2e.log.dir}/0.log</File>
<rollingPolicy class="ch.qos.logback.core.rolling.FixedWindowRollingPolicy">
<FileNamePattern>${org.eclipse.m2e.log.dir}/%i.log</FileNamePattern>
<MinIndex>1</MinIndex>
<MaxIndex>10</MaxIndex>
</rollingPolicy>
<triggeringPolicy class="ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy">
<MaxFileSize>10MB</MaxFileSize>
</triggeringPolicy>
<encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
<pattern>%date [%thread] %-5level %logger{35} - %msg%n</pattern>
</encoder>
</appender>
<appender name="EclipseLog" class="org.eclipse.m2e.logback.appender.EclipseLogAppender">
<filter class="ch.qos.logback.classic.filter.ThresholdFilter">
<level>WARN</level>
</filter>
</appender>
<appender name="MavenConsoleLog" class="org.eclipse.m2e.logback.appender.MavenConsoleAppender">
</appender>
<root level="INFO">
<appender-ref ref="FILE" />
<appender-ref ref="STDOUT" />
<appender-ref ref="EclipseLog" />
<appender-ref ref="MavenConsoleLog" />
</root>
</configuration>
@@ -3,4 +3,5 @@
<workingSet editPageId="org.eclipse.jdt.internal.ui.DynamicSourcesWorkingSet" factoryID="org.eclipse.ui.internal.WorkingSetFactory" id="1785979555150_0" label="Java Main Sources" name="Java Main Sources"/>
<workingSet editPageId="org.eclipse.jdt.internal.ui.DynamicSourcesWorkingSet" factoryID="org.eclipse.ui.internal.WorkingSetFactory" id="1785979555168_1" label="Java Test Sources" name="Java Test Sources"/>
<workingSet aggregate="true" factoryID="org.eclipse.ui.internal.WorkingSetFactory" id="1785979556746_2" label="Window Working Set" name="Aggregate for window 1785979556746"/>
<workingSet aggregate="true" factoryID="org.eclipse.ui.internal.WorkingSetFactory" id="1791324496308_3" label="Window Working Set" name="Aggregate for window 1791324496308"/>
</workingSetManager>