ssboost commited on
Commit
2bb1715
ยท
verified ยท
1 Parent(s): 74c61e9

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +274 -419
app.py CHANGED
@@ -1,426 +1,281 @@
1
- import spaces
2
- import math
3
  import gradio as gr
4
- import numpy as np
5
- import torch
6
- import safetensors.torch as sf
7
-
8
- from PIL import Image
9
- from diffusers import StableDiffusionPipeline, StableDiffusionImg2ImgPipeline
10
- from diffusers import AutoencoderKL, UNet2DConditionModel, DDIMScheduler, EulerAncestralDiscreteScheduler, DPMSolverMultistepScheduler
11
- from diffusers.models.attention_processor import AttnProcessor2_0
12
- from transformers import CLIPTextModel, CLIPTokenizer
13
- from briarmbg import BriaRMBG
14
- from enum import Enum
15
-
16
- # Cohere ๋ชจ๋ธ์„ ์‚ฌ์šฉํ•˜๊ธฐ ์œ„ํ•œ ๋ผ์ด๋ธŒ๋Ÿฌ๋ฆฌ
17
  from huggingface_hub import InferenceClient
18
 
19
- # CohereForAI/c4ai-command-r-plus ๋ชจ๋ธ ์‚ฌ์šฉ ์„ค์ •
20
- client = InferenceClient(model_name="CohereForAI/c4ai-command-r-plus")
21
-
22
- # Stable Diffusion ์„ค์ •
23
- sd15_name = 'stablediffusionapi/realistic-vision-v51'
24
- model_path = './models/iclight_sd15_fc.safetensors'
25
-
26
- # ๋ชจ๋ธ ๋กœ๋“œ
27
- tokenizer = CLIPTokenizer.from_pretrained(sd15_name, subfolder="tokenizer")
28
- text_encoder = CLIPTextModel.from_pretrained(sd15_name, subfolder="text_encoder")
29
- vae = AutoencoderKL.from_pretrained(sd15_name, subfolder="vae")
30
- unet = UNet2DConditionModel.from_pretrained(sd15_name, subfolder="unet")
31
- rmbg = BriaRMBG.from_pretrained("briaai/RMBG-1.4")
32
-
33
- # UNet ์ˆ˜์ •
34
- with torch.no_grad():
35
- new_conv_in = torch.nn.Conv2d(8, unet.conv_in.out_channels, unet.conv_in.kernel_size, unet.conv_in.stride, unet.conv_in.padding)
36
- new_conv_in.weight.zero_()
37
- new_conv_in.weight[:, :4, :, :].copy_(unet.conv_in.weight)
38
- new_conv_in.bias = unet.conv_in.bias
39
- unet.conv_in = new_conv_in
40
-
41
- unet_original_forward = unet.forward
42
-
43
- def hooked_unet_forward(sample, timestep, encoder_hidden_states, **kwargs):
44
- c_concat = kwargs.get('cross_attention_kwargs', {}).get('concat_conds', None)
45
- if c_concat is not None:
46
- c_concat = c_concat.to(sample.device)
47
- c_concat = torch.cat([c_concat] * (sample.shape[0] // c_concat.shape[0]), dim=0)
48
- sample = torch.cat([sample, c_concat], dim=1)
49
- kwargs['cross_attention_kwargs'] = {}
50
- return unet_original_forward(sample, timestep, encoder_hidden_states, **kwargs)
51
-
52
- unet.forward = hooked_unet_forward
53
-
54
- # ๋ชจ๋ธ ๋กœ๋“œ
55
- try:
56
- sd_offset = sf.load_file(model_path)
57
- sd_origin = unet.state_dict()
58
- sd_merged = {k: sd_origin[k] + sd_offset[k] for k in sd_origin.keys()}
59
- unet.load_state_dict(sd_merged, strict=True)
60
- del sd_offset, sd_origin, sd_merged
61
- except FileNotFoundError:
62
- print(f"Error: Model file not found at {model_path}")
63
- # ์ ์ ˆํ•œ ์˜ˆ์™ธ ์ฒ˜๋ฆฌ๋ฅผ ์ถ”๊ฐ€ํ•  ์ˆ˜ ์žˆ์Šต๋‹ˆ๋‹ค.
64
-
65
- # ๋””๋ฐ”์ด์Šค ์„ค์ •
66
- device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
67
- text_encoder = text_encoder.to(device=device, dtype=torch.float16)
68
- vae = vae.to(device=device, dtype=torch.float16)
69
- unet = unet.to(device=device, dtype=torch.float16)
70
- rmbg = rmbg.to(device=device, dtype=torch.float32)
71
-
72
- # SDP ์„ค์ •
73
- unet.set_attn_processor(AttnProcessor2_0())
74
- vae.set_attn_processor(AttnProcessor2_0())
75
-
76
- # ์Šค์ผ€์ค„๋Ÿฌ ์„ค์ •
77
- ddim_scheduler = DDIMScheduler(
78
- num_train_timesteps=1000,
79
- beta_start=0.00085,
80
- beta_end=0.012,
81
- beta_schedule="scaled_linear",
82
- clip_sample=False,
83
- set_alpha_to_one=False,
84
- steps_offset=1,
85
- )
86
-
87
- euler_a_scheduler = EulerAncestralDiscreteScheduler(
88
- num_train_timesteps=1000,
89
- beta_start=0.00085,
90
- beta_end=0.012,
91
- steps_offset=1
92
- )
93
-
94
- dpmpp_2m_sde_karras_scheduler = DPMSolverMultistepScheduler(
95
- num_train_timesteps=1000,
96
- beta_start=0.00085,
97
- beta_end=0.012,
98
- algorithm_type="sde-dpmsolver++",
99
- use_karras_sigmas=True,
100
- steps_offset=1
101
- )
102
-
103
- # ํŒŒ์ดํ”„๋ผ์ธ ์„ค์ •
104
- t2i_pipe = StableDiffusionPipeline(
105
- vae=vae,
106
- text_encoder=text_encoder,
107
- tokenizer=tokenizer,
108
- unet=unet,
109
- scheduler=dpmpp_2m_sde_karras_scheduler,
110
- safety_checker=None,
111
- requires_safety_checker=False,
112
- feature_extractor=None,
113
- image_encoder=None
114
- )
115
-
116
- i2i_pipe = StableDiffusionImg2ImgPipeline(
117
- vae=vae,
118
- text_encoder=text_encoder,
119
- tokenizer=tokenizer,
120
- unet=unet,
121
- scheduler=dpmpp_2m_sde_karras_scheduler,
122
- safety_checker=None,
123
- requires_safety_checker=False,
124
- feature_extractor=None,
125
- image_encoder=None
126
- )
127
-
128
- @torch.inference_mode()
129
- def encode_prompt_inner(txt: str):
130
- max_length = tokenizer.model_max_length
131
- chunk_length = tokenizer.model_max_length - 2
132
- id_start = tokenizer.bos_token_id
133
- id_end = tokenizer.eos_token_id
134
- id_pad = id_end
135
-
136
- def pad(x, p, i):
137
- return x[:i] if len(x) >= i else x + [p] * (i - len(x))
138
-
139
- tokens = tokenizer(txt, truncation=False, add_special_tokens=False)["input_ids"]
140
- chunks = [[id_start] + tokens[i: i + chunk_length] + [id_end] for i in range(0, len(tokens), chunk_length)]
141
- chunks = [pad(ck, id_pad, max_length) for ck in chunks]
142
-
143
- token_ids = torch.tensor(chunks).to(device=device, dtype=torch.int64)
144
- conds = text_encoder(token_ids).last_hidden_state
145
-
146
- return conds
147
-
148
- @torch.inference_mode()
149
- def encode_prompt_pair(positive_prompt, negative_prompt):
150
- c = encode_prompt_inner(positive_prompt)
151
- uc = encode_prompt_inner(negative_prompt)
152
-
153
- c_len = float(len(c))
154
- uc_len = float(len(uc))
155
- max_count = max(c_len, uc_len)
156
- c_repeat = int(math.ceil(max_count / c_len))
157
- uc_repeat = int(math.ceil(max_count / uc_len))
158
- max_chunk = max(len(c), len(uc))
159
-
160
- c = torch.cat([c] * c_repeat, dim=0)[:max_chunk]
161
- uc = torch.cat([uc] * uc_repeat, dim=0)[:max_chunk]
162
-
163
- c = torch.cat([p[None, ...] for p in c], dim=1)
164
- uc = torch.cat([p[None, ...] for p in uc], dim=1)
165
-
166
- return c, uc
167
-
168
- @torch.inference_mode()
169
- def pytorch2numpy(imgs, quant=True):
170
- results = []
171
- for x in imgs:
172
- y = x.movedim(0, -1)
173
-
174
- if quant:
175
- y = y * 127.5 + 127.5
176
- y = y.detach().float().cpu().numpy().clip(0, 255).astype(np.uint8)
177
  else:
178
- y = y * 0.5 + 0.5
179
- y = y.detach().float().cpu().numpy().clip(0, 1).astype(np.float32)
180
-
181
- results.append(y)
182
- return results
183
-
184
- @torch.inference_mode()
185
- def numpy2pytorch(imgs):
186
- h = torch.from_numpy(np.stack(imgs, axis=0)).float() / 127.0 - 1.0 # so that 127 must be strictly 0.0
187
- h = h.movedim(-1, 1)
188
- return h
189
-
190
- def resize_and_center_crop(image, target_width, target_height):
191
- pil_image = Image.fromarray(image)
192
- original_width, original_height = pil_image.size
193
- scale_factor = max(target_width / original_width, target_height / original_height)
194
- resized_width = int(round(original_width * scale_factor))
195
- resized_height = int(round(original_height * scale_factor))
196
- resized_image = pil_image.resize((resized_width, resized_height), Image.LANCZOS)
197
- left = (resized_width - target_width) / 2
198
- top = (resized_height - target_height) / 2
199
- right = (resized_width + target_width) / 2
200
- bottom = (resized_height + target_height) / 2
201
- cropped_image = resized_image.crop((left, top, right, bottom))
202
- return np.array(cropped_image)
203
-
204
- def resize_without_crop(image, target_width, target_height):
205
- pil_image = Image.fromarray(image)
206
- resized_image = pil_image.resize((target_width, target_height), Image.LANCZOS)
207
- return np.array(resized_image)
208
-
209
- @torch.inference_mode()
210
- def run_rmbg(img, sigma=0.0):
211
- H, W, C = img.shape
212
- assert C == 3
213
- k = (256.0 / float(H * W)) ** 0.5
214
- feed = resize_without_crop(img, int(64 * round(W * k)), int(64 * round(H * k)))
215
- feed = numpy2pytorch([feed]).to(device=device, dtype=torch.float32)
216
- alpha = rmbg(feed)[0][0]
217
- alpha = torch.nn.functional.interpolate(alpha, size=(H, W), mode="bilinear")
218
- alpha = alpha.movedim(1, -1)[0]
219
- alpha = alpha.detach().float().cpu().numpy().clip(0, 1)
220
- result = 127 + (img.astype(np.float32) - 127 + sigma) * alpha
221
- return result.clip(0, 255).astype(np.uint8), alpha
222
-
223
- @torch.inference_mode()
224
- def process(input_fg, prompt, bg_source, image_width, image_height):
225
- # UI์—์„œ ์ œ๊ฑฐ๋œ ๋งค๊ฐœ๋ณ€์ˆ˜์˜ ๊ฐ’ ์„ค์ •
226
- num_samples = 1
227
- seed = 12345
228
- steps = 25
229
- cfg = 2
230
- lowres_denoise = 0.9
231
- highres_scale = 1.5
232
- highres_denoise = 0.5
233
- a_prompt = 'best quality'
234
- n_prompt = 'lowres, bad anatomy, bad hands, cropped, worst quality'
235
-
236
- bg_source = BGSource(bg_source)
237
- input_bg = None
238
-
239
- # ์ด๋ฏธ์ง€ ํฌ๊ธฐ๋ฅผ 8์˜ ๋ฐฐ์ˆ˜๋กœ ์กฐ์ •
240
- image_width = (image_width // 8) * 8
241
- image_height = (image_height // 8) * 8
242
-
243
- if bg_source == BGSource.NONE:
244
- pass
245
- elif bg_source == BGSource.LEFT:
246
- gradient = np.linspace(255, 0, image_width)
247
- image = np.tile(gradient, (image_height, 1))
248
- input_bg = np.stack((image,) * 3, axis=-1).astype(np.uint8)
249
- elif bg_source == BGSource.RIGHT:
250
- gradient = np.linspace(0, 255, image_width)
251
- image = np.tile(gradient, (image_height, 1))
252
- input_bg = np.stack((image,) * 3, axis=-1).astype(np.uint8)
253
- elif bg_source == BGSource.TOP:
254
- gradient = np.linspace(255, 0, image_height)[:, None]
255
- image = np.tile(gradient, (1, image_width))
256
- input_bg = np.stack((image,) * 3, axis=-1).astype(np.uint8)
257
- elif bg_source == BGSource.BOTTOM:
258
- gradient = np.linspace(0, 255, image_height)[:, None]
259
- image = np.tile(gradient, (1, image_width))
260
- input_bg = np.stack((image,) * 3, axis=-1).astype(np.uint8)
261
- else:
262
- raise ValueError('Wrong initial latent!')
263
-
264
- rng = torch.Generator(device=device).manual_seed(int(seed))
265
-
266
- fg = resize_and_center_crop(input_fg, image_width, image_height)
267
-
268
- concat_conds = numpy2pytorch([fg]).to(device=vae.device, dtype=vae.dtype)
269
- concat_conds = vae.encode(concat_conds).latent_dist.mode() * vae.config.scaling_factor
270
-
271
- conds, unconds = encode_prompt_pair(positive_prompt=prompt + ', ' + a_prompt, negative_prompt=n_prompt)
272
-
273
- if input_bg is None:
274
- latents = t2i_pipe(
275
- prompt_embeds=conds,
276
- negative_prompt_embeds=unconds,
277
- width=image_width,
278
- height=image_height,
279
- num_inference_steps=steps,
280
- num_images_per_prompt=num_samples,
281
- generator=rng,
282
- output_type='latent',
283
- guidance_scale=cfg,
284
- cross_attention_kwargs={'concat_conds': concat_conds},
285
- ).images.to(vae.dtype) / vae.config.scaling_factor
286
- else:
287
- bg = resize_and_center_crop(input_bg, image_width, image_height)
288
- bg_latent = numpy2pytorch([bg]).to(device=vae.device, dtype=vae.dtype)
289
- bg_latent = vae.encode(bg_latent).latent_dist.mode() * vae.config.scaling_factor
290
- latents = i2i_pipe(
291
- image=bg_latent,
292
- strength=lowres_denoise,
293
- prompt_embeds=conds,
294
- negative_prompt_embeds=unconds,
295
- width=image_width,
296
- height=image_height,
297
- num_inference_steps=int(round(steps / lowres_denoise)),
298
- num_images_per_prompt=num_samples,
299
- generator=rng,
300
- output_type='latent',
301
- guidance_scale=cfg,
302
- cross_attention_kwargs={'concat_conds': concat_conds},
303
- ).images.to(vae.dtype) / vae.config.scaling_factor
304
-
305
- pixels = vae.decode(latents).sample
306
- pixels = pytorch2numpy(pixels)
307
- pixels = [resize_without_crop(
308
- image=p,
309
- target_width=int(round(image_width * highres_scale / 64.0) * 64),
310
- target_height=int(round(image_height * highres_scale / 64.0) * 64))
311
- for p in pixels]
312
-
313
- pixels = numpy2pytorch(pixels).to(device=vae.device, dtype=vae.dtype)
314
- latents = vae.encode(pixels).latent_dist.mode() * vae.config.scaling_factor
315
- latents = latents.to(device=unet.device, dtype=unet.dtype)
316
-
317
- image_height, image_width = latents.shape[2] * 8, latents.shape[3] * 8
318
-
319
- fg = resize_and_center_crop(input_fg, image_width, image_height)
320
- concat_conds = numpy2pytorch([fg]).to(device=vae.device, dtype=vae.dtype)
321
- concat_conds = vae.encode(concat_conds).latent_dist.mode() * vae.config.scaling_factor
322
-
323
- latents = i2i_pipe(
324
- image=latents,
325
- strength=highres_denoise,
326
- prompt_embeds=conds,
327
- negative_prompt_embeds=unconds,
328
- width=image_width,
329
- height=image_height,
330
- num_inference_steps=int(round(steps / highres_denoise)),
331
- num_images_per_prompt=num_samples,
332
- generator=rng,
333
- output_type='latent',
334
- guidance_scale=cfg,
335
- cross_attention_kwargs={'concat_conds': concat_conds},
336
- ).images.to(vae.dtype) / vae.config.scaling_factor
337
-
338
- pixels = vae.decode(latents).sample
339
-
340
- return pytorch2numpy(pixels)
341
-
342
- @spaces.GPU
343
- @torch.inference_mode()
344
- def process_relight(input_fg, prompt, bg_source):
345
- input_fg, matting = run_rmbg(input_fg)
346
- image_width, image_height = input_fg.shape[1], input_fg.shape[0]
347
- results = process(input_fg, prompt, bg_source, image_width, image_height)
348
-
349
- # ์ด๋ฏธ์ง€ ์ €์žฅ
350
- output_image_path = "/tmp/output_image.png"
351
- Image.fromarray(results[0]).save(output_image_path)
352
-
353
- return input_fg, results, output_image_path
354
-
355
- # ํ•œ๊ตญ์–ด ํ…์ŠคํŠธ๋ฅผ Stable Diffusion ํ”„๋กฌํ”„ํŠธ๋กœ ๋ณ€ํ™˜
356
- def korean_to_sd_prompt(korean_text):
357
- system_message = "์ฃผ์–ด์ง„ ํ•œ๊ตญ์–ด ํ…์ŠคํŠธ๋ฅผ Stable Diffusion ํ”„๋กฌํ”„ํŠธ๋กœ ๋ณ€ํ™˜ํ•˜์„ธ์š”. ํ”„๋กฌํ”„ํŠธ๋Š” ์ฃผ๋กœ ์‚ฌ์šฉ๋˜๋Š” ํ‚ค์›Œ๋“œ๋“ค๋กœ ๊ตฌ์„ฑ๋œ ํ˜•ํƒœ์—ฌ์•ผ ํ•ฉ๋‹ˆ๋‹ค."
358
- response = client.chat_completion(
359
- messages=[
360
- {"role": "system", "content": system_message},
361
- {"role": "user", "content": korean_text}
362
- ],
363
- max_tokens=100
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
364
  )
365
- return response.choices[0].message.content.strip()
366
-
367
- quick_prompts = [
368
- 'sunshine from window',
369
- 'neon light, city',
370
- 'sunset over sea',
371
- 'golden time',
372
- 'sci-fi RGB glowing, cyberpunk',
373
- 'natural lighting',
374
- 'warm atmosphere, at home, bedroom',
375
- 'magic lit',
376
- 'evil, gothic, Yharnam',
377
- 'light and shadow',
378
- 'shadow from window',
379
- 'soft studio lighting',
380
- 'home atmosphere, cozy bedroom illumination',
381
- 'neon, Wong Kar-wai, warm'
382
- ]
383
- quick_prompts = [[x] for x in quick_prompts]
384
-
385
- quick_subjects = [
386
- 'beautiful woman, detailed face',
387
- 'handsome man, detailed face',
388
- ]
389
- quick_subjects = [[x] for x in quick_subjects]
390
-
391
- class BGSource(Enum):
392
- NONE = "None"
393
- LEFT = "Left Light"
394
- RIGHT = "Right Light"
395
- TOP = "Top Light"
396
- BOTTOM = "Bottom Light"
397
-
398
- block = gr.Blocks().queue()
399
- with block:
400
- with gr.Row():
401
- gr.Markdown("## IC-Light (Relighting with Foreground Condition)")
402
- with gr.Row():
403
- gr.Markdown("See also https://github.com/lllyasviel/IC-Light for background-conditioned model and normal estimation")
404
- with gr.Row():
405
- with gr.Column():
406
- with gr.Row():
407
- input_fg = gr.Image(sources='upload', type="numpy", label="Image", height=480)
408
- output_bg = gr.Image(type="numpy", label="Preprocessed Foreground", height=480, visible=False) # UI์—์„œ ๋ณด์ด์ง€ ์•Š๋„๋ก ์„ค์ •
409
- prompt = gr.Textbox(label="Prompt")
410
- bg_source = gr.Radio(choices=[e.value for e in BGSource],
411
- value=BGSource.NONE.value,
412
- label="Lighting Preference (Initial Latent)", type='value')
413
- example_quick_subjects = gr.Dataset(samples=quick_subjects, label='Subject Quick List', samples_per_page=1000, components=[prompt])
414
- example_quick_prompts = gr.Dataset(samples=quick_prompts, label='Lighting Quick List', samples_per_page=1000, components=[prompt])
415
- relight_button = gr.Button(value="Relight")
416
-
417
- with gr.Column():
418
- result_gallery = gr.Gallery(height=832, object_fit='contain', label='Outputs')
419
- download_button = gr.File(label="Download Processed Image")
420
 
421
- ips = [input_fg, prompt, bg_source]
422
- relight_button.click(fn=process_relight, inputs=ips, outputs=[output_bg, result_gallery, download_button])
423
- example_quick_prompts.click(lambda x, y: ', '.join(y.split(', ')[:2] + [x[0]]), inputs=[example_quick_prompts, prompt], outputs=prompt, show_progress=False, queue=False)
424
- example_quick_subjects.click(lambda x: x[0], inputs=example_quick_subjects, outputs=prompt, show_progress=False, queue=False)
425
 
426
- block.launch(server_name='0.0.0.0')
 
 
 
1
  import gradio as gr
2
+ import requests
3
+ from bs4 import BeautifulSoup
4
+ from requests.adapters import HTTPAdapter
5
+ from requests.packages.urllib3.util.retry import Retry
6
+ import re
7
+ import time
8
+ import random
9
+ import os
 
 
 
 
 
10
  from huggingface_hub import InferenceClient
11
 
12
+ def setup_session():
13
+ try:
14
+ session = requests.Session()
15
+ retries = Retry(total=5, backoff_factor=1, status_forcelist=[502, 503, 504])
16
+ session.mount('https://', HTTPAdapter(max_retries=retries))
17
+ return session
18
+ except Exception as e:
19
+ return None
20
+
21
+ def generate_naver_search_url(query):
22
+ base_url = "https://search.naver.com/search.naver?"
23
+ params = {"ssc": "tab.blog.all", "sm": "tab_jum", "query": query}
24
+ url = base_url + "&".join(f"{key}={value}" for key, value in params.items())
25
+ return url
26
+
27
+ def crawl_blog_content(url, session):
28
+ try:
29
+ headers = {
30
+ "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3",
31
+ "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8",
32
+ "Accept-Language": "ko-KR,ko;q=0.9,en-US;q=0.8,en;q=0.7",
33
+ "Accept-Encoding": "gzip, deflate, br",
34
+ "Connection": "keep-alive",
35
+ "Referer": "https://search.naver.com/search.naver",
36
+ }
37
+
38
+ # ๋žœ๋ค ๋”œ๋ ˆ์ด ์ถ”๊ฐ€
39
+ delay = random.uniform(1, 2)
40
+ time.sleep(delay)
41
+
42
+ response = session.get(url, headers=headers)
43
+ if response.status_code != 200:
44
+ return ""
45
+
46
+ soup = BeautifulSoup(response.content, "html.parser")
47
+ content = soup.find("div", attrs={'class': 'se-main-container'})
48
+
49
+ if content:
50
+ return clean_text(content.get_text())
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
51
  else:
52
+ return ""
53
+ except Exception as e:
54
+ return ""
55
+
56
+ def crawl_naver_search_results(url, session):
57
+ try:
58
+ headers = {
59
+ "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3",
60
+ "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8",
61
+ "Accept-Language": "ko-KR,ko;q=0.9,en-US;q=0.8,en;q=0.7",
62
+ "Accept-Encoding": "gzip, deflate, br",
63
+ "Connection": "keep-alive",
64
+ "Referer": "https://search.naver.com/search.naver",
65
+ }
66
+ response = session.get(url, headers=headers)
67
+ if response.status_code != 200:
68
+ return []
69
+
70
+ soup = BeautifulSoup(response.content, "html.parser")
71
+ results = []
72
+ count = 0
73
+ for li in soup.find_all("li", class_=re.compile("bx.*")):
74
+ if count >= 10:
75
+ break
76
+ for div in li.find_all("div", class_="detail_box"):
77
+ for div2 in div.find_all("div", class_="title_area"):
78
+ title = div2.text.strip()
79
+ for a in div2.find_all("a", href=True):
80
+ link = a["href"]
81
+ if "blog.naver" in link:
82
+ link = link.replace("https://", "https://m.")
83
+ results.append({"์ œ๋ชฉ": title, "๋งํฌ": link})
84
+ count += 1
85
+ if count >= 10:
86
+ break
87
+ if count >= 10:
88
+ break
89
+ if count >= 10:
90
+ break
91
+
92
+ return results
93
+ except Exception as e:
94
+ return []
95
+
96
+ def clean_text(text):
97
+ text = re.sub(r'\s+', ' ', text).strip()
98
+ return text
99
+
100
+ def create_client(model_name):
101
+ return InferenceClient(model_name, token=os.getenv("HF_TOKEN"))
102
+
103
+ client = create_client("CohereForAI/c4ai-command-r-plus")
104
+
105
+ def call_api(content, system_message, max_tokens, temperature, top_p):
106
+ messages = [{"role": "system", "content": system_message}, {"role": "user", "content": content}]
107
+ random_seed = random.randint(0, 1000000)
108
+ response = client.chat_completion(messages=messages, max_tokens=max_tokens, temperature=temperature, top_p=top_p, seed=random_seed)
109
+ modified_text = response.choices[0].message.content
110
+ input_tokens = response.usage.prompt_tokens
111
+ output_tokens = response.usage.completion_tokens
112
+ total_tokens = response.usage.total_tokens
113
+ return modified_text, input_tokens, output_tokens, total_tokens
114
+
115
+ def analyze_info(category, topic, references1, references2, references3):
116
+ return f"์„ ํƒํ•œ ์นดํ…Œ๊ณ ๋ฆฌ: {category}\n๋ธ”๋กœ๊ทธ ์ฃผ์ œ: {topic}\n์ฐธ๊ณ  ๊ธ€1: {references1}\n์ฐธ๊ณ  ๊ธ€2: {references2}\n์ฐธ๊ณ  ๊ธ€3: {references3}"
117
+
118
+ def suggest_title(category, topic, references1, references2, references3, system_message, max_tokens, temperature, top_p):
119
+ full_content = analyze_info(category, topic, references1, references2, references3)
120
+ modified_text, input_tokens, output_tokens, total_tokens = call_api(full_content, system_message, max_tokens, temperature, top_p)
121
+ token_usage_message = f"[์ž…๋ ฅ ํ† ํฐ์ˆ˜: {input_tokens}]\n[์ถœ๋ ฅ ํ† ํฐ์ˆ˜: {output_tokens}]\n[์ด ํ† ํฐ์ˆ˜: {total_tokens}]"
122
+ return modified_text, token_usage_message
123
+
124
+ def generate_outline(category, topic, references1, references2, references3, title, system_message, max_tokens, temperature, top_p):
125
+ full_content = analyze_info(category, topic, references1, references2, references3)
126
+ content = f"{full_content}\nTitle: {title}"
127
+ modified_text, input_tokens, output_tokens, total_tokens = call_api(content, system_message, max_tokens, temperature, top_p)
128
+ token_usage_message = f"[์ž…๋ ฅ ํ† ํฐ์ˆ˜: {input_tokens}]\n[์ถœ๋ ฅ ํ† ํฐ์ˆ˜: {output_tokens}]\n[์ด ํ† ํฐ์ˆ˜: {total_tokens}]"
129
+ return modified_text, token_usage_message
130
+
131
+ def generate_blog_post(category, topic, references1, references2, references3, title, outline, system_message, max_tokens, temperature, top_p):
132
+ full_content = analyze_info(category, topic, references1, references2, references3)
133
+ content = f"{full_content}\nTitle: {title}\nOutline: {outline}"
134
+ modified_text, input_tokens, output_tokens, total_tokens = call_api(content, system_message, max_tokens, temperature, top_p)
135
+ formatted_text = modified_text.replace('\n', '\n\n')
136
+ token_usage_message = f"[์ž…๋ ฅ ํ† ํฐ์ˆ˜: {input_tokens}]\n[์ถœ๋ ฅ ํ† ํฐ์ˆ˜: {output_tokens}]\n[์ด ํ† ํฐ์ˆ˜: {total_tokens}]"
137
+ return formatted_text, token_usage_message
138
+
139
+ def fetch_references(topic):
140
+ search_url = generate_naver_search_url(topic)
141
+ session = setup_session()
142
+ if session is None:
143
+ return "Failed to set up session.", "", "", ""
144
+ results = crawl_naver_search_results(search_url, session)
145
+ if not results:
146
+ return "No results found.", "", "", ""
147
+
148
+ selected_results = random.sample(results, 3)
149
+ references1_content = f"์ œ๋ชฉ: {selected_results[0]['์ œ๋ชฉ']}\n๋‚ด์šฉ: {crawl_blog_content(selected_results[0]['๋งํฌ'], session)}"
150
+ references2_content = f"์ œ๋ชฉ: {selected_results[1]['์ œ๋ชฉ']}\n๋‚ด์šฉ: {crawl_blog_content(selected_results[1]['๋งํฌ'], session)}"
151
+ references3_content = f"์ œ๋ชฉ: {selected_results[2]['์ œ๋ชฉ']}\n๋‚ด์šฉ: {crawl_blog_content(selected_results[2]['๋งํฌ'], session)}"
152
+
153
+ return "์ฐธ๊ณ ๊ธ€ ์ƒ์„ฑ ์™„๋ฃŒ", references1_content, references2_content, references3_content
154
+
155
+ def fetch_references_and_generate_all_steps(category, topic, blog_title, system_message_outline, max_tokens_outline, temperature_outline, top_p_outline, system_message_blog_post, max_tokens_blog_post, temperature_blog_post, top_p_blog_post):
156
+ search_url = generate_naver_search_url(topic)
157
+ session = setup_session()
158
+ if session is None:
159
+ return "", "", "", "", "", "", "", "", "", ""
160
+
161
+ results = crawl_naver_search_results(search_url, session)
162
+ if not results:
163
+ return "", "", "", "", "", "", "", "", "", ""
164
+
165
+ selected_results = random.sample(results, 3)
166
+ references1_content = f"์ œ๋ชฉ: {selected_results[0]['์ œ๋ชฉ']}\n๋‚ด์šฉ: {crawl_blog_content(selected_results[0]['๋งํฌ'], session)}"
167
+ references2_content = f"์ œ๋ชฉ: {selected_results[1]['์ œ๋ชฉ']}\n๋‚ด์šฉ: {crawl_blog_content(selected_results[1]['๋งํฌ'], session)}"
168
+ references3_content = f"์ œ๋ชฉ: {selected_results[2]['์ œ๋ชฉ']}\n๋‚ด์šฉ: {crawl_blog_content(selected_results[2]['๋งํฌ'], session)}"
169
+
170
+ # ์•„์›ƒ๋ผ์ธ ์ƒ์„ฑ
171
+ outline_result, outline_token_usage = generate_outline(category, topic, references1_content, references2_content, references3_content, blog_title, system_message_outline, max_tokens_outline, temperature_outline, top_p_outline)
172
+
173
+ # ๋ธ”๋กœ๊ทธ ๊ธ€ ์ƒ์„ฑ
174
+ blog_post_result, blog_post_token_usage = generate_blog_post(category, topic, references1_content, references2_content, references3_content, blog_title, outline_result, system_message_blog_post, max_tokens_blog_post, temperature_blog_post, top_p_blog_post)
175
+
176
+ return references1_content, references2_content, references3_content, outline_result, outline_token_usage, blog_post_result, blog_post_token_usage
177
+
178
+ def get_title_prompt(category):
179
+ if (category == "์ผ๋ฐ˜"):
180
+ return """
181
+ # ๋ธ”๋กœ๊ทธ ์ œ๋ชฉ ์ƒ์„ฑ ๊ทœ์น™(์ผ๋ฐ˜)
182
+ """
183
+ elif (category == "๊ฑด๊ฐ•์ •๋ณด"):
184
+ return """
185
+ # ๋ธ”๋กœ๊ทธ ์ œ๋ชฉ ์ƒ์„ฑ ๊ทœ์น™(๊ฑด๊ฐ•์ •๋ณด)
186
+ """
187
+
188
+ def get_outline_prompt(category):
189
+ if (category == "์ผ๋ฐ˜"):
190
+ return """
191
+ # ๋ธ”๋กœ๊ทธ ์†Œ์ฃผ์ œ(Subtopic) ์ƒ์„ฑ ๊ทœ์น™(์ผ๋ฐ˜)
192
+ """
193
+ elif (category == "๊ฑด๊ฐ•์ •๋ณด"):
194
+ return """
195
+ # ๋ธ”๋กœ๊ทธ ์†Œ์ฃผ์ œ(Subtopic) ์ƒ์„ฑ ๊ทœ์น™(๊ฑด๊ฐ•์ •๋ณด)
196
+ """
197
+
198
+ def get_blog_post_prompt(category):
199
+ if (category == "์ผ๋ฐ˜"):
200
+ return """
201
+ # ๋ธ”๋กœ๊ทธ ํ…์ŠคํŠธ ์ƒ์„ฑ ๊ทœ์น™(์ผ๋ฐ˜)
202
+ """
203
+ elif (category == "๊ฑด๊ฐ•์ •๋ณด"):
204
+ return """
205
+ # ๋ธ”๋กœ๊ทธ ํ…์ŠคํŠธ ์ƒ์„ฑ ๊ทœ์น™(๊ฑด๊ฐ•์ •๋ณด)
206
+ """
207
+
208
+ # Gradio ์ธํ„ฐํŽ˜์ด์Šค ๊ตฌ์„ฑ
209
+ title = "์ •๋ณด์„ฑ ํฌ์ŠคํŒ… ์ž๋™์ƒ์„ฑ๊ธฐ(์ œ๋ชฉ์ถ”์ฒœ ํ›„ ์ž๋™)"
210
+
211
+ def update_prompts(category):
212
+ title_prompt = get_title_prompt(category)
213
+ outline_prompt = get_outline_prompt(category)
214
+ blog_post_prompt = get_blog_post_prompt(category)
215
+ return title_prompt, outline_prompt, blog_post_prompt
216
+
217
+ with gr.Blocks() as demo:
218
+ gr.Markdown(f"# {title}")
219
+
220
+ # 1๋‹จ๊ณ„
221
+ gr.Markdown("### 1๋‹จ๊ณ„ : ํฌ์ŠคํŒ… ์นดํ…Œ๊ณ ๋ฆฌ๋ฅผ ์ง€์ •ํ•ด์ฃผ์„ธ์š”")
222
+ category = gr.Radio(choices=["์ผ๋ฐ˜", "๊ฑด๊ฐ•์ •๋ณด"], label="ํฌ์ŠคํŒ… ์นดํ…Œ๊ณ ๋ฆฌ", value="์ผ๋ฐ˜")
223
+
224
+ # 2๋‹จ๊ณ„
225
+ gr.Markdown("### 2๋‹จ๊ณ„ : ๋ธ”๋กœ๊ทธ ์ฃผ์ œ, ๋˜๋Š” ํ‚ค์›Œ๋“œ๋ฅผ ์ƒ์„ธํžˆ ์ž…๋ ฅํ•˜์„ธ์š”")
226
+ topic = gr.Textbox(label="๋ธ”๋กœ๊ทธ ์ฃผ์ œ(์˜ˆ์‹œ: ์˜ค์ง•์–ด ๋ฌด์นจํšŒ(X), ์˜ค์ง•์–ด ๋ฌด์นจํšŒ ๋ ˆ์‹œํ”ผ(O))", placeholder="์˜ˆ์‹œ: ์—ฌํ–‰์ง€ ์ถ”์ฒœ(X), 8์›” ๊ตญ๋‚ด ์—ฌํ–‰์ง€ ์ถ”์ฒœ(O)")
227
+
228
+ # 3๋‹จ๊ณ„: ์ฐธ๊ณ  ๊ธ€์„ ์œ„ํ•œ ๋ณ€์ˆ˜๋“ค ๋ฏธ๋ฆฌ ์ •์˜
229
+ references1 = gr.Textbox(label="์ฐธ๊ณ  ๊ธ€ 1", placeholder="์ฐธ๊ณ ํ•  ๋ธ”๋กœ๊ทธ ํฌ์ŠคํŒ…๊ธ€์„ ๋ณต์‚ฌํ•˜์—ฌ ๋ถ™์—ฌ๋„ฃ์œผ์„ธ์š”", lines=10, visible=False)
230
+ references2 = gr.Textbox(label="์ฐธ๊ณ  ๊ธ€ 2", placeholder="์ฐธ๊ณ ํ•  ๋ธ”๋กœ๊ทธ ํฌ์ŠคํŒ…๊ธ€์„ ๋ณต์‚ฌํ•˜์—ฌ ๋ถ™์—ฌ๋„ฃ์œผ์„ธ์š”", lines=10, visible=False)
231
+ references3 = gr.Textbox(label="์ฐธ๊ณ  ๊ธ€ 3", placeholder="์ฐธ๊ณ ํ•  ๋ธ”๋กœ๊ทธ ํฌ์ŠคํŒ…๊ธ€์„ ๋ณต์‚ฌํ•˜์—ฌ ๋ถ™์—ฌ๋„ฃ์œผ์„ธ์š”", lines=10, visible=False)
232
+
233
+ # ์ œ๋ชฉ ์ถ”์ฒœ
234
+ gr.Markdown("### 4๋‹จ๊ณ„ : ์ œ๋ชฉ ์ถ”์ฒœํ•˜๊ธฐ")
235
+
236
+ with gr.Accordion("์ œ๋ชฉ ์„ค์ •", open=True):
237
+ title_system_message = gr.Textbox(label="์‹œ์Šคํ…œ ๋ฉ”์‹œ์ง€", value=get_title_prompt("์ผ๋ฐ˜"), lines=15)
238
+ title_max_tokens = gr.Slider(label="Max Tokens", minimum=1000, maximum=8000, value=5000, step=1000)
239
+ title_temperature = gr.Slider(label="Temperature", minimum=0.1, maximum=1.0, value=0.8, step=0.1)
240
+ title_top_p = gr.Slider(label="Top P", minimum=0.1, maximum=1.0, value=0.95, step=0.05)
241
+
242
+ title_suggestions = gr.Textbox(label="์ œ๋ชฉ ์ถ”์ฒœ", lines=10)
243
+ title_token_output = gr.Markdown(label="์‚ฌ์šฉ๋œ ํ† ํฐ ์ˆ˜")
244
+
245
+ # ์ œ๋ชฉ ์ถ”์ฒœ ๋ฒ„ํŠผ
246
+ title_btn = gr.Button("์ œ๋ชฉ ์ถ”์ฒœํ•˜๊ธฐ")
247
+ title_btn.click(fn=suggest_title, inputs=[category, topic, references1, references2, references3, title_system_message, title_max_tokens, title_temperature, title_top_p], outputs=[title_suggestions, title_token_output])
248
+
249
+ blog_title = gr.Textbox(label="๋ธ”๋กœ๊ทธ ์ œ๋ชฉ", placeholder="๋ธ”๋กœ๊ทธ ์ œ๋ชฉ์„ ์ž…๋ ฅํ•ด์ฃผ์„ธ์š”")
250
+
251
+ # ๋ธ”๋กœ๊ทธ ๊ธ€ ์ƒ์„ฑ
252
+ gr.Markdown("### 5๋‹จ๊ณ„ : ๋ธ”๋กœ๊ทธ ๊ธ€ ์ƒ์„ฑํ•˜๊ธฐ")
253
+ gr.HTML("<span style='color: grey;'>[๋ธ”๋กœ๊ทธ ๊ธ€ ์ƒ์„ฑํ•˜๊ธฐ ๋ฒ„ํŠผ์„ ํด๋ฆญํ•˜๋ฉด ์•„์›ƒ๋ผ์ธ ์ƒ์„ฑ ๋ฐ ๋ธ”๋กœ๊ทธ ๊ธ€ ์ž‘์„ฑ์ด ์ž๋™์œผ๋กœ ์ง„ํ–‰๋ฉ๋‹ˆ๋‹ค.]</span>")
254
+
255
+ with gr.Accordion("๋ธ”๋กœ๊ทธ ๊ธ€ ์„ค์ •", open=True):
256
+ outline_system_message = gr.Textbox(label="์‹œ์Šคํ…œ ๋ฉ”์‹œ์ง€", value=get_outline_prompt("์ผ๋ฐ˜"), lines=20)
257
+ outline_max_tokens = gr.Slider(label="Max Tokens", minimum=1000, maximum=8000, value=6000, step=1000)
258
+ outline_temperature = gr.Slider(label="Temperature", minimum=0.1, maximum=1.0, value=0.8, step=0.1)
259
+ outline_top_p = gr.Slider(label="Top P", minimum=0.1, maximum=1.0, value=0.95, step=0.05)
260
+
261
+ blog_system_message = gr.Textbox(label="์‹œ์Šคํ…œ ๋ฉ”์‹œ์ง€", value=get_blog_post_prompt("์ผ๋ฐ˜"), lines=20)
262
+ blog_max_tokens = gr.Slider(label="Max Tokens", minimum=1000, maximum=12000, value=8000, step=1000)
263
+ blog_temperature = gr.Slider(label="Temperature", minimum=0.1, maximum=1.0, value=0.8, step=0.1)
264
+ blog_top_p = gr.Slider(label="Top P", minimum=0.1, maximum=1.0, value=0.95, step=0.05)
265
+
266
+ outline_result = gr.Textbox(label="์•„์›ƒ๋ผ์ธ ๊ฒฐ๊ณผ", lines=15, visible=False)
267
+ outline_token_output = gr.Markdown(label="์‚ฌ์šฉ๋œ ํ† ํฐ ์ˆ˜", visible=False)
268
+ output = gr.Textbox(label="์ƒ์„ฑ๋œ ๋ธ”๋กœ๊ทธ ๊ธ€", lines=30)
269
+ token_output = gr.Markdown(label="์‚ฌ์šฉ๋œ ํ† ํฐ ์ˆ˜")
270
+
271
+ # ๋ธ”๋กœ๊ทธ ๊ธ€ ์ƒ์„ฑ ๋ฒ„ํŠผ
272
+ generate_post_btn = gr.Button("๋ธ”๋กœ๊ทธ ๊ธ€ ์ƒ์„ฑํ•˜๊ธฐ")
273
+ generate_post_btn.click(
274
+ fn=fetch_references_and_generate_all_steps,
275
+ inputs=[category, topic, blog_title, outline_system_message, outline_max_tokens, outline_temperature, outline_top_p, blog_system_message, blog_max_tokens, blog_temperature, blog_top_p],
276
+ outputs=[references1, references2, references3, outline_result, outline_token_output, output, token_output]
277
  )
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
278
 
279
+ category.change(fn=update_prompts, inputs=category, outputs=[title_system_message, outline_system_message, blog_system_message])
 
 
 
280
 
281
+ demo.launch()