From 3c77ed3c5fd5386c7d2c5ebc31ac3b9dc698d73a Mon Sep 17 00:00:00 2001 From: zhaoqichao <1416537683@qq.com> Date: Wed, 5 Jul 2023 09:29:21 +0800 Subject: [PATCH] =?UTF-8?q?Url=E8=A7=A3=E6=9E=90=E7=BB=84=E4=BB=B6?= =?UTF-8?q?=E9=87=8D=E6=9E=84=E5=8F=8A=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ...Discover.java => AbstractUrlDiscover.java} | 38 +++++++----- .../utils/discover/CommonUrlDiscover.java | 56 +++++++++++++++++ .../discover/CommonUrlTitleDiscover.java | 15 ----- .../discover/PrioritizedUrlDiscover.java | 60 +++++++++++++++++++ .../discover/PrioritizedUrlTitleDiscover.java | 33 ---------- .../common/utils/discover/UrlDiscover.java | 46 ++++++++++++++ .../utils/discover/UrlTitleDiscover.java | 35 ----------- .../common/utils/discover/WxUrlDiscover.java | 30 ++++++++++ .../utils/discover/WxUrlTitleDiscover.java | 15 ----- .../service/strategy/msg/TextMsgHandler.java | 4 +- 10 files changed, 218 insertions(+), 114 deletions(-) rename mallchat-common/src/main/java/com/abin/mallchat/common/common/utils/discover/{AbstractUrlTitleDiscover.java => AbstractUrlDiscover.java} (64%) create mode 100644 mallchat-common/src/main/java/com/abin/mallchat/common/common/utils/discover/CommonUrlDiscover.java delete mode 100644 mallchat-common/src/main/java/com/abin/mallchat/common/common/utils/discover/CommonUrlTitleDiscover.java create mode 100644 mallchat-common/src/main/java/com/abin/mallchat/common/common/utils/discover/PrioritizedUrlDiscover.java delete mode 100644 mallchat-common/src/main/java/com/abin/mallchat/common/common/utils/discover/PrioritizedUrlTitleDiscover.java create mode 100644 mallchat-common/src/main/java/com/abin/mallchat/common/common/utils/discover/UrlDiscover.java delete mode 100644 mallchat-common/src/main/java/com/abin/mallchat/common/common/utils/discover/UrlTitleDiscover.java create mode 100644 mallchat-common/src/main/java/com/abin/mallchat/common/common/utils/discover/WxUrlDiscover.java delete mode 100644 mallchat-common/src/main/java/com/abin/mallchat/common/common/utils/discover/WxUrlTitleDiscover.java diff --git a/mallchat-common/src/main/java/com/abin/mallchat/common/common/utils/discover/AbstractUrlTitleDiscover.java b/mallchat-common/src/main/java/com/abin/mallchat/common/common/utils/discover/AbstractUrlDiscover.java similarity index 64% rename from mallchat-common/src/main/java/com/abin/mallchat/common/common/utils/discover/AbstractUrlTitleDiscover.java rename to mallchat-common/src/main/java/com/abin/mallchat/common/common/utils/discover/AbstractUrlDiscover.java index 79cbefe..80d7ab8 100644 --- a/mallchat-common/src/main/java/com/abin/mallchat/common/common/utils/discover/AbstractUrlTitleDiscover.java +++ b/mallchat-common/src/main/java/com/abin/mallchat/common/common/utils/discover/AbstractUrlDiscover.java @@ -3,14 +3,15 @@ package com.abin.mallchat.common.common.utils.discover; import cn.hutool.core.util.ReUtil; import cn.hutool.core.util.StrUtil; import com.abin.mallchat.common.common.utils.FutureUtils; +import com.abin.mallchat.common.common.utils.discover.domain.UrlInfo; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; +import org.jetbrains.annotations.Nullable; import org.jsoup.Connection; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.springframework.data.util.Pair; -import javax.annotation.Nullable; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -20,46 +21,55 @@ import java.util.regex.Pattern; import java.util.stream.Collectors; /** - * Description: urlTitle查询抽象类 - * Author: abin - * Date: 2023-05-27 + * @author zhaoqichao + * @date 2023/7/3 16:38 */ @Slf4j -public abstract class AbstractUrlTitleDiscover implements UrlTitleDiscover { +public abstract class AbstractUrlDiscover implements UrlDiscover { //链接识别的正则 private static final Pattern PATTERN = Pattern.compile("((http|https)://)?(www.)?([\\w_-]+(?:(?:\\.[\\w_-]+)+))([\\w.,@?^=%&:/~+#-]*[\\w@?^=%&/~+#-])?"); - @Nullable + + @javax.annotation.Nullable @Override - public Map getContentTitleMap(String content) { + public Map getUrlContentMap(String content) { + if (StrUtil.isBlank(content)) { return new HashMap<>(); } List matchList = ReUtil.findAll(PATTERN, content, 0); + //并行请求 - List>> futures = matchList.stream().map(match -> CompletableFuture.supplyAsync(() -> { - String title = getUrlTitle(match); - return StringUtils.isNotEmpty(title) ? Pair.of(match, title) : null; + List>> futures = matchList.stream().map(match -> CompletableFuture.supplyAsync(() -> { + UrlInfo urlInfo = getContent(match); + return Objects.isNull(urlInfo) ? null : Pair.of(match, urlInfo); })).collect(Collectors.toList()); - CompletableFuture>> future = FutureUtils.sequenceNonNull(futures); + CompletableFuture>> future = FutureUtils.sequenceNonNull(futures); //结果组装 return future.join().stream().collect(Collectors.toMap(Pair::getFirst, Pair::getSecond, (a, b) -> a)); } @Nullable @Override - public String getUrlTitle(String url) { + public UrlInfo getContent(String url) { Document document = getUrlDocument(assemble(url)); if (Objects.isNull(document)) { return null; } - return getDocTitle(document); + + return UrlInfo.builder() + .title(getTitle(document)) + .description(getDescription(document)) + .image(getImage(assemble(url),document)).build(); } + private String assemble(String url) { + if (!StrUtil.startWith(url, "http")) { return "http://" + url; } + return url; } @@ -69,7 +79,7 @@ public abstract class AbstractUrlTitleDiscover implements UrlTitleDiscover { connect.timeout(2000); return connect.get(); } catch (Exception e) { - log.error("find title error:url:{}", matchUrl, e); + log.error("find error:url:{}", matchUrl, e); } return null; } diff --git a/mallchat-common/src/main/java/com/abin/mallchat/common/common/utils/discover/CommonUrlDiscover.java b/mallchat-common/src/main/java/com/abin/mallchat/common/common/utils/discover/CommonUrlDiscover.java new file mode 100644 index 0000000..b73c81d --- /dev/null +++ b/mallchat-common/src/main/java/com/abin/mallchat/common/common/utils/discover/CommonUrlDiscover.java @@ -0,0 +1,56 @@ +package com.abin.mallchat.common.common.utils.discover; + +import cn.hutool.core.util.StrUtil; +import io.jsonwebtoken.lang.Objects; +import org.apache.commons.lang3.StringUtils; +import org.jetbrains.annotations.Nullable; +import org.jsoup.nodes.Document; +import org.jsoup.select.Elements; + +/** + * @author zhaoqichao + * @date 2023/7/3 16:54 + */ +public class CommonUrlDiscover extends AbstractUrlDiscover { + @Nullable + @Override + public String getTitle(Document document) { + return document.title(); + } + + @Nullable + @Override + public String getDescription(Document document) { + String description = document.head().select("meta[name=description]").attr("content"); + String keywords = document.head().select("meta[name=keywords]").attr("content"); + String content = StrUtil.isNotBlank(description) ? description : keywords; + //只保留一句话的描述 + return StrUtil.isNotBlank(content) ? content.substring(0, content.indexOf("。")) : content; + } + + @Nullable + @Override + public String getImage(String url, Document document) { + //如果包含og则是微信链接 + if(StrUtil.isNotBlank(document.getElementsByAttributeValue("property", "og:title").attr("content"))){ + return null; + } + String image = document.select("link[type=image/x-icon]").attr("href"); + //如果没有去匹配含有icon属性的logo + String href = StrUtil.isEmpty(image) ? document.select("link[rel$=icon]").attr("href") : image; + //如果icon中已经包含了url部分域名 + if (StrUtil.isNotBlank(StrUtil.removeAny(StrUtil.removeAny(href, "/"), "favicon.ico")) && + StrUtil.containsAny(StrUtil.removePrefix(url, "http://"), StrUtil.removeAny(StrUtil.removeAny(href, "/"), "favicon.ico"))) { + return "http://" + StrUtil.removePrefix(href, "/"); + } + //如果url已经包含了logo + if (StrUtil.containsAny(url, "favicon")) { + return url; + } + //如果logo中有url + if (StrUtil.containsAny(href, "http") || StrUtil.containsAny(href, "https")) { + return href; + } + return StrUtil.format("{}/{}", url, StrUtil.removePrefix(href, "/")); + } +} diff --git a/mallchat-common/src/main/java/com/abin/mallchat/common/common/utils/discover/CommonUrlTitleDiscover.java b/mallchat-common/src/main/java/com/abin/mallchat/common/common/utils/discover/CommonUrlTitleDiscover.java deleted file mode 100644 index 6471610..0000000 --- a/mallchat-common/src/main/java/com/abin/mallchat/common/common/utils/discover/CommonUrlTitleDiscover.java +++ /dev/null @@ -1,15 +0,0 @@ -package com.abin.mallchat.common.common.utils.discover; - -import org.jsoup.nodes.Document; - -/** - * Description: 通用的标题解析类 - * Author: abin - * Date: 2023-05-27 - */ -public class CommonUrlTitleDiscover extends AbstractUrlTitleDiscover { - @Override - public String getDocTitle(Document document) { - return document.title(); - } -} diff --git a/mallchat-common/src/main/java/com/abin/mallchat/common/common/utils/discover/PrioritizedUrlDiscover.java b/mallchat-common/src/main/java/com/abin/mallchat/common/common/utils/discover/PrioritizedUrlDiscover.java new file mode 100644 index 0000000..ab57334 --- /dev/null +++ b/mallchat-common/src/main/java/com/abin/mallchat/common/common/utils/discover/PrioritizedUrlDiscover.java @@ -0,0 +1,60 @@ +package com.abin.mallchat.common.common.utils.discover; + +import cn.hutool.core.util.StrUtil; +import org.jetbrains.annotations.Nullable; +import org.jsoup.nodes.Document; + +import java.util.ArrayList; +import java.util.List; + +/** + * Description: 具有优先级的title查询器 + * Author: abin + * Date: 2023-05-27 + */ +public class PrioritizedUrlDiscover extends AbstractUrlDiscover { + + private final List urlTitleDiscovers = new ArrayList<>(2); + + public PrioritizedUrlDiscover() { + urlTitleDiscovers.add(new CommonUrlDiscover()); + urlTitleDiscovers.add(new WxUrlDiscover()); + } + + + @Nullable + @Override + public String getTitle(Document document) { + for (UrlDiscover urlDiscover : urlTitleDiscovers) { + String urlTitle = urlDiscover.getTitle(document); + if (StrUtil.isNotBlank(urlTitle)) { + return urlTitle; + } + } + return null; + } + + @Nullable + @Override + public String getDescription(Document document) { + for (UrlDiscover urlDiscover : urlTitleDiscovers) { + String urlDescription = urlDiscover.getDescription(document); + if (StrUtil.isNotBlank(urlDescription)) { + return urlDescription; + } + } + return null; + } + + @Nullable + @Override + public String getImage(String url, Document document) { + for (UrlDiscover urlDiscover : urlTitleDiscovers) { + String urlImage = urlDiscover.getImage(url,document); + if (StrUtil.isNotBlank(urlImage)) { + return urlImage; + } + } + return null; + } +} diff --git a/mallchat-common/src/main/java/com/abin/mallchat/common/common/utils/discover/PrioritizedUrlTitleDiscover.java b/mallchat-common/src/main/java/com/abin/mallchat/common/common/utils/discover/PrioritizedUrlTitleDiscover.java deleted file mode 100644 index 8c7fb4d..0000000 --- a/mallchat-common/src/main/java/com/abin/mallchat/common/common/utils/discover/PrioritizedUrlTitleDiscover.java +++ /dev/null @@ -1,33 +0,0 @@ -package com.abin.mallchat.common.common.utils.discover; - -import cn.hutool.core.util.StrUtil; -import org.jsoup.nodes.Document; - -import java.util.ArrayList; -import java.util.List; - -/** - * Description: 具有优先级的title查询器 - * Author: abin - * Date: 2023-05-27 - */ -public class PrioritizedUrlTitleDiscover extends AbstractUrlTitleDiscover { - - private final List urlTitleDiscovers = new ArrayList<>(2); - - public PrioritizedUrlTitleDiscover() { - urlTitleDiscovers.add(new CommonUrlTitleDiscover()); - urlTitleDiscovers.add(new WxUrlTitleDiscover()); - } - - @Override - public String getDocTitle(Document document) { - for (UrlTitleDiscover urlTitleDiscover : urlTitleDiscovers) { - String urlTitle = urlTitleDiscover.getDocTitle(document); - if (StrUtil.isNotBlank(urlTitle)) { - return urlTitle; - } - } - return null; - } -} diff --git a/mallchat-common/src/main/java/com/abin/mallchat/common/common/utils/discover/UrlDiscover.java b/mallchat-common/src/main/java/com/abin/mallchat/common/common/utils/discover/UrlDiscover.java new file mode 100644 index 0000000..3515c28 --- /dev/null +++ b/mallchat-common/src/main/java/com/abin/mallchat/common/common/utils/discover/UrlDiscover.java @@ -0,0 +1,46 @@ +package com.abin.mallchat.common.common.utils.discover; + +import cn.hutool.core.date.StopWatch; +import cn.hutool.core.util.StrUtil; +import com.abin.mallchat.common.common.utils.discover.domain.UrlInfo; +import org.jsoup.nodes.Document; + +import javax.annotation.Nullable; +import java.util.Map; + +/** + * @author zhaoqichao + * @date 2023/7/3 16:34 + */ +public interface UrlDiscover { + + + @Nullable + Map getUrlContentMap(String content); + + @Nullable + UrlInfo getContent(String url); + + @Nullable + String getTitle(Document document); + + @Nullable + String getDescription(Document document); + + @Nullable + String getImage(String url, Document document); + + public static void main(String[] args) { + StopWatch stopWatch = new StopWatch(); + stopWatch.start(); + String longStr = "其中包含一个URL www.baidu.com,一个带有端口号的URL http://www.jd.com:80, 一个带有路径的URL http://mallchat.cn, 还有美团技术文章https://mp.weixin.qq.com/s/hwTf4bDck9_tlFpgVDeIKg "; +// String longStr = "一个带有端口号的URL http://www.jd.com:80,"; +// String longStr = "一个带有路径的URL http://mallchat.cn"; + PrioritizedUrlDiscover discover = new PrioritizedUrlDiscover(); + final Map map = discover.getUrlContentMap(longStr); + System.out.println(map); + stopWatch.stop(); + long cost = stopWatch.getTotalTimeMillis(); + System.out.println(cost); + } +} diff --git a/mallchat-common/src/main/java/com/abin/mallchat/common/common/utils/discover/UrlTitleDiscover.java b/mallchat-common/src/main/java/com/abin/mallchat/common/common/utils/discover/UrlTitleDiscover.java deleted file mode 100644 index e2fac68..0000000 --- a/mallchat-common/src/main/java/com/abin/mallchat/common/common/utils/discover/UrlTitleDiscover.java +++ /dev/null @@ -1,35 +0,0 @@ -package com.abin.mallchat.common.common.utils.discover; - -import cn.hutool.core.date.StopWatch; -import org.jsoup.nodes.Document; - -import javax.annotation.Nullable; -import java.util.Map; - -public interface UrlTitleDiscover { - - - @Nullable - Map getContentTitleMap(String content); - - - @Nullable - String getUrlTitle(String url); - - @Nullable - String getDocTitle(Document document); - - public static void main(String[] args) {//用异步多任务查询并合并 974 //串行访问的速度1349 1291 1283 1559 - StopWatch stopWatch = new StopWatch(); - stopWatch.start(); - String longStr = "这是一个很长的字符串再来 www.github.com，其中包含一个URL www.baidu.com,, 一个带有端口号的URL http://www.jd.com:80, 一个带有路径的URL http://mallchat.cn, 还有美团技术文章https://mp.weixin.qq.com/s/hwTf4bDck9_tlFpgVDeIKg "; - PrioritizedUrlTitleDiscover discover = new PrioritizedUrlTitleDiscover(); - Map contentTitleMap = discover.getContentTitleMap(longStr); - System.out.println(contentTitleMap); -// -// Jsoup.connect("http:// www.github.com"); - stopWatch.stop(); - long cost = stopWatch.getTotalTimeMillis(); - System.out.println(cost); - }//{http://mallchat.cn=MallChat, www.baidu.com=百度一下，你就知道, https://mp.weixin.qq.com/s/hwTf4bDck9_tlFpgVDeIKg=超大规模数据库集群保稳系列之二：数据库攻防演练建设实践, http://www.jd.com:80=京东(JD.COM)-正品低价、品质保障、配送及时、轻松购物！} -} diff --git a/mallchat-common/src/main/java/com/abin/mallchat/common/common/utils/discover/WxUrlDiscover.java b/mallchat-common/src/main/java/com/abin/mallchat/common/common/utils/discover/WxUrlDiscover.java new file mode 100644 index 0000000..a5bb330 --- /dev/null +++ b/mallchat-common/src/main/java/com/abin/mallchat/common/common/utils/discover/WxUrlDiscover.java @@ -0,0 +1,30 @@ +package com.abin.mallchat.common.common.utils.discover; + +import org.jetbrains.annotations.Nullable; +import org.jsoup.nodes.Document; + +/** + * Description: 针对微信公众号文章的标题获取类 + * Author: abin + * Date: 2023-05-27 + */ +public class WxUrlDiscover extends AbstractUrlDiscover { + + @Nullable + @Override + public String getTitle(Document document) { + return document.getElementsByAttributeValue("property", "og:title").attr("content"); + } + + @Nullable + @Override + public String getDescription(Document document) { + return document.getElementsByAttributeValue("property", "og:description").attr("content"); + } + + @Nullable + @Override + public String getImage(String url, Document document) { + return document.getElementsByAttributeValue("property", "og:image").attr("content"); + } +} diff --git a/mallchat-common/src/main/java/com/abin/mallchat/common/common/utils/discover/WxUrlTitleDiscover.java b/mallchat-common/src/main/java/com/abin/mallchat/common/common/utils/discover/WxUrlTitleDiscover.java deleted file mode 100644 index 29b1172..0000000 --- a/mallchat-common/src/main/java/com/abin/mallchat/common/common/utils/discover/WxUrlTitleDiscover.java +++ /dev/null @@ -1,15 +0,0 @@ -package com.abin.mallchat.common.common.utils.discover; - -import org.jsoup.nodes.Document; - -/** - * Description: 针对微信公众号文章的标题获取类 - * Author: abin - * Date: 2023-05-27 - */ -public class WxUrlTitleDiscover extends AbstractUrlTitleDiscover { - @Override - public String getDocTitle(Document document) { - return document.getElementsByAttributeValue("property", "og:title").attr("content"); - } -} diff --git a/mallchat-custom-server/src/main/java/com/abin/mallchat/custom/chat/service/strategy/msg/TextMsgHandler.java b/mallchat-custom-server/src/main/java/com/abin/mallchat/custom/chat/service/strategy/msg/TextMsgHandler.java index d8e347a..6b63a31 100644 --- a/mallchat-custom-server/src/main/java/com/abin/mallchat/custom/chat/service/strategy/msg/TextMsgHandler.java +++ b/mallchat-custom-server/src/main/java/com/abin/mallchat/custom/chat/service/strategy/msg/TextMsgHandler.java @@ -11,7 +11,7 @@ import com.abin.mallchat.common.chat.service.cache.MsgCache; import com.abin.mallchat.common.common.domain.enums.YesOrNoEnum; import com.abin.mallchat.common.common.utils.AssertUtil; import com.abin.mallchat.common.common.utils.SensitiveWordUtils; -import com.abin.mallchat.common.common.utils.discover.PrioritizedUrlTitleDiscover; +import com.abin.mallchat.common.common.utils.discover.PrioritizedUrlDiscover; import com.abin.mallchat.common.user.domain.entity.User; import com.abin.mallchat.common.user.domain.enums.RoleEnum; import com.abin.mallchat.common.user.service.IRoleService; @@ -47,7 +47,7 @@ public class TextMsgHandler extends AbstractMsgHandler { @Autowired private IRoleService iRoleService; - private static final PrioritizedUrlTitleDiscover URL_TITLE_DISCOVER = new PrioritizedUrlTitleDiscover(); + private static final PrioritizedUrlDiscover URL_TITLE_DISCOVER = new PrioritizedUrlDiscover(); @Override MessageTypeEnum getMsgTypeEnum() {